Bug 1025066 - Update to ssl-supporting wptrunner, a=testonly
authorJames Graham <james@hoppipolla.co.uk>
Sat, 10 Jan 2015 10:42:43 +0000
changeset 223115 47671b5f66ab8182bebfe255f946b500cab65908
parent 223114 9de247dc316174bd7c35d1e7449a82a5318cc153
child 223116 d16c7eac45de738cfbee6c902ea41a87232232e7
push id28082
push usercbook@mozilla.com
push dateMon, 12 Jan 2015 10:44:52 +0000
treeherdermozilla-central@643589c3ef94 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1025066
milestone37.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1025066 - Update to ssl-supporting wptrunner, a=testonly
testing/web-platform/harness/requirements.txt
testing/web-platform/harness/setup.py
testing/web-platform/harness/wptrunner/browsers/b2g.py
testing/web-platform/harness/wptrunner/browsers/base.py
testing/web-platform/harness/wptrunner/browsers/chrome.py
testing/web-platform/harness/wptrunner/browsers/firefox.py
testing/web-platform/harness/wptrunner/browsers/server-locations.txt
testing/web-platform/harness/wptrunner/browsers/servo.py
testing/web-platform/harness/wptrunner/config.json
testing/web-platform/harness/wptrunner/executors/executorservo.py
testing/web-platform/harness/wptrunner/products.py
testing/web-platform/harness/wptrunner/wptcommandline.py
testing/web-platform/harness/wptrunner/wptrunner.py
--- a/testing/web-platform/harness/requirements.txt
+++ b/testing/web-platform/harness/requirements.txt
@@ -1,5 +1,5 @@
 html5lib >= 0.99
 mozinfo >= 0.7
-mozlog >= 1.8
+mozlog >= 2.8
 # Unfortunately, just for gdb flags
 mozrunner >= 6.1
--- a/testing/web-platform/harness/setup.py
+++ b/testing/web-platform/harness/setup.py
@@ -7,17 +7,17 @@ import os
 import sys
 import textwrap
 
 from setuptools import setup, find_packages
 
 here = os.path.split(__file__)[0]
 
 PACKAGE_NAME = 'wptrunner'
-PACKAGE_VERSION = '1.7'
+PACKAGE_VERSION = '1.8'
 
 # Dependencies
 with open(os.path.join(here, "requirements.txt")) as f:
     deps = f.read().splitlines()
 
 # Browser-specific requirements
 requirements_files = glob.glob(os.path.join(here, "requirements_*.txt"))
 
--- a/testing/web-platform/harness/wptrunner/browsers/b2g.py
+++ b/testing/web-platform/harness/wptrunner/browsers/b2g.py
@@ -31,17 +31,17 @@ here = os.path.split(__file__)[0]
                  "executor_kwargs": "executor_kwargs",
                  "env_options": "env_options"}
 
 
 def check_args(**kwargs):
     pass
 
 
-def browser_kwargs(**kwargs):
+def browser_kwargs(test_environment, **kwargs):
     return {"prefs_root": kwargs["prefs_root"],
             "no_backup": kwargs.get("b2g_no_backup", False)}
 
 
 def executor_kwargs(http_server_url, **kwargs):
     timeout_multiplier = kwargs["timeout_multiplier"]
     if timeout_multiplier is None:
         timeout_multiplier = 2
--- a/testing/web-platform/harness/wptrunner/browsers/base.py
+++ b/testing/web-platform/harness/wptrunner/browsers/base.py
@@ -90,16 +90,20 @@ class Browser(object):
         """pid of the browser process or None if there is no pid"""
         pass
 
     @abstractmethod
     def is_alive(self):
         """Boolean indicating whether the browser process is still running"""
         pass
 
+    def setup_ssl(self, hosts):
+        """Return a certificate to use for tests requiring ssl that will be trusted by the browser"""
+        raise NotImplementedError("ssl testing not supported")
+
     def cleanup(self):
         """Browser-specific cleanup that is run after the testrun is finished"""
         pass
 
     def executor_browser(self):
         """Returns the ExecutorBrowser subclass for this Browser subclass and the keyword arguments
         with which it should be instantiated"""
         return ExecutorBrowser, {}
--- a/testing/web-platform/harness/wptrunner/browsers/chrome.py
+++ b/testing/web-platform/harness/wptrunner/browsers/chrome.py
@@ -36,17 +36,17 @@ def executor_kwargs(http_server_url, **k
                         {"chromeOptions": {"binary": binary}}.items())
 
     return {"http_server_url": http_server_url,
             "capabilities": capabilities,
             "timeout_multiplier": timeout_multiplier}
 
 
 def env_options():
-    return {"host": "localhost",
+    return {"host": "web-platform.test",
             "bind_hostname": "true",
             "required_files": required_files}
 
 
 class ChromeBrowser(Browser):
     """Chrome is backed by chromedriver, which is supplied through
     ``browsers.webdriver.ChromedriverLocalServer``."""
 
--- a/testing/web-platform/harness/wptrunner/browsers/firefox.py
+++ b/testing/web-platform/harness/wptrunner/browsers/firefox.py
@@ -1,14 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
+import subprocess
 
+import mozinfo
 from mozprocess import ProcessHandler
 from mozprofile import FirefoxProfile, Preferences
 from mozprofile.permissions import ServerLocations
 from mozrunner import FirefoxRunner
 from mozcrash import mozcrash
 
 from .base import get_free_port, Browser, ExecutorBrowser, require_arg, cmd_arg
 from ..executors import executor_kwargs as base_executor_kwargs
@@ -23,56 +25,65 @@ here = os.path.join(os.path.split(__file
                               "reftest": "MarionetteReftestExecutor"},
                  "browser_kwargs": "browser_kwargs",
                  "executor_kwargs": "executor_kwargs",
                  "env_options": "env_options"}
 
 
 def check_args(**kwargs):
     require_arg(kwargs, "binary")
+    if kwargs["ssl_type"] != "none":
+        require_arg(kwargs, "certutil_binary")
 
 
 def browser_kwargs(**kwargs):
     return {"binary": kwargs["binary"],
             "prefs_root": kwargs["prefs_root"],
             "debug_args": kwargs["debug_args"],
             "interactive": kwargs["interactive"],
-            "symbols_path":kwargs["symbols_path"],
-            "stackwalk_binary":kwargs["stackwalk_binary"]}
+            "symbols_path": kwargs["symbols_path"],
+            "stackwalk_binary": kwargs["stackwalk_binary"],
+            "certutil_binary": kwargs["certutil_binary"],
+            "ca_certificate_path": kwargs["ssl_env"].ca_cert_path()}
 
 
 def executor_kwargs(http_server_url, **kwargs):
     executor_kwargs = base_executor_kwargs(http_server_url, **kwargs)
     executor_kwargs["close_after_done"] = True
     return executor_kwargs
 
 
 def env_options():
-    return {"host": "localhost",
+    return {"host": "127.0.0.1",
             "external_host": "web-platform.test",
-            "bind_hostname": "true",
-            "required_files": required_files}
+            "bind_hostname": "false",
+            "required_files": required_files,
+            "certificate_domain": "web-platform.test",
+            "encrypt_after_connect": True}
 
 
 class FirefoxBrowser(Browser):
     used_ports = set()
 
     def __init__(self, logger, binary, prefs_root, debug_args=None, interactive=None,
-                 symbols_path=None, stackwalk_binary=None):
+                 symbols_path=None, stackwalk_binary=None, certutil_binary=None,
+                 ca_certificate_path=None):
         Browser.__init__(self, logger)
         self.binary = binary
         self.prefs_root = prefs_root
         self.marionette_port = None
         self.used_ports.add(self.marionette_port)
         self.runner = None
         self.debug_args = debug_args
         self.interactive = interactive
         self.profile = None
         self.symbols_path = symbols_path
         self.stackwalk_binary = stackwalk_binary
+        self.ca_certificate_path = ca_certificate_path
+        self.certutil_binary = certutil_binary
 
     def start(self):
         self.marionette_port = get_free_port(2828, exclude=self.used_ports)
 
         env = os.environ.copy()
         env["MOZ_CRASHREPORTER"] = "1"
         env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
         env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
@@ -88,16 +99,19 @@ class FirefoxBrowser(Browser):
 
         self.profile = FirefoxProfile(locations=locations,
                                       proxy=ports,
                                       preferences=preferences)
         self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
                                       "marionette.defaultPrefs.port": self.marionette_port,
                                       "dom.disable_open_during_load": False})
 
+        if self.ca_certificate_path is not None:
+            self.setup_ssl()
+
         self.runner = FirefoxRunner(profile=self.profile,
                                     binary=self.binary,
                                     cmdargs=[cmd_arg("marionette"), "about:blank"],
                                     env=env,
                                     process_class=ProcessHandler,
                                     process_args={"processOutputLine": [self.on_output]})
 
         self.logger.debug("Starting Firefox")
@@ -147,11 +161,55 @@ class FirefoxBrowser(Browser):
         self.stop()
 
     def executor_browser(self):
         assert self.marionette_port is not None
         return ExecutorBrowser, {"marionette_port": self.marionette_port}
 
     def log_crash(self, process, test):
         dump_dir = os.path.join(self.profile.profile, "minidumps")
-        mozcrash.log_crashes(self.logger, dump_dir, symbols_path=self.symbols_path,
+
+        mozcrash.log_crashes(self.logger,
+                             dump_dir,
+                             symbols_path=self.symbols_path,
                              stackwalk_binary=self.stackwalk_binary,
-                             process=process, test=test)
+                             process=process,
+                             test=test)
+
+    def setup_ssl(self):
+        """Create a certificate database to use in the test profile. This is configured
+        to trust the CA Certificate that has signed the web-platform.test server
+        certificate."""
+
+        self.logger.info("Setting up ssl")
+
+        # Make sure the certutil libraries from the source tree are loaded when using a
+        # local copy of certutil
+        # TODO: Maybe only set this if certutil won't launch?
+        env = os.environ.copy()
+        certutil_dir = os.path.dirname(self.binary)
+        env["LD_LIBRARY_PATH"] = certutil_dir
+        env["PATH"] = os.path.pathsep.join([certutil_dir, env["PATH"]])
+
+        def certutil(*args):
+            cmd = [self.certutil_binary] + list(args)
+            self.logger.process_output("certutil",
+                                       subprocess.check_output(cmd,
+                                                               env=env,
+                                                               stderr=subprocess.STDOUT),
+                                       " ".join(cmd))
+
+        pw_path = os.path.join(self.profile.profile, ".crtdbpw")
+        with open(pw_path, "w") as f:
+            # Use empty password for certificate db
+            f.write("\n")
+
+        cert_db_path = self.profile.profile
+
+        # Create a new certificate db
+        certutil("-N", "-d", cert_db_path, "-f", pw_path)
+
+        # Add the CA certificate to the database and mark as trusted to issue server certs
+        certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,",
+                 "-n", "web-platform-tests", "-i", self.ca_certificate_path)
+
+        # List all certs in the database
+        certutil("-L", "-d", cert_db_path)
--- a/testing/web-platform/harness/wptrunner/browsers/server-locations.txt
+++ b/testing/web-platform/harness/wptrunner/browsers/server-locations.txt
@@ -16,16 +16,23 @@ http://xn--lve-6lad.web-platform.test:80
 
 http://web-platform.test:8001
 http://www.web-platform.test:8001
 http://www1.web-platform.test:8001
 http://www2.web-platform.test:8001
 http://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8001
 http://xn--lve-6lad.web-platform.test:8001
 
+https://web-platform.test:8443
+https://www.web-platform.test:8443
+https://www1.web-platform.test:8443
+https://www2.web-platform.test:8443
+https://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8443
+https://xn--lve-6lad.web-platform.test:8443
+
 # These are actually ws servers, but until mozprofile is
 # fixed we have to pretend that they are http servers
 http://web-platform.test:8888
 http://www.web-platform.test:8888
 http://www1.web-platform.test:8888
 http://www2.web-platform.test:8888
 http://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8888
 http://xn--lve-6lad.web-platform.test:8888
--- a/testing/web-platform/harness/wptrunner/browsers/servo.py
+++ b/testing/web-platform/harness/wptrunner/browsers/servo.py
@@ -1,24 +1,25 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
 
 from .base import NullBrowser, ExecutorBrowser, require_arg
 from ..executors import executor_kwargs
-from ..executors.executorservo import ServoTestharnessExecutor
+from ..executors.executorservo import ServoTestharnessExecutor, ServoReftestExecutor
 
 here = os.path.join(os.path.split(__file__)[0])
 
 __wptrunner__ = {"product": "servo",
                  "check_args": "check_args",
                  "browser": "ServoBrowser",
-                 "executor": {"testharness": "ServoTestharnessExecutor"},
+                 "executor": {"testharness": "ServoTestharnessExecutor",
+                              "reftest": "ServoReftestExecutor"},
                  "browser_kwargs": "browser_kwargs",
                  "executor_kwargs": "executor_kwargs",
                  "env_options": "env_options"}
 
 
 def check_args(**kwargs):
     require_arg(kwargs, "binary")
 
--- a/testing/web-platform/harness/wptrunner/config.json
+++ b/testing/web-platform/harness/wptrunner/config.json
@@ -1,6 +1,7 @@
 {"host": "%(host)s",
  "ports":{"http":[8000, 8001],
-          "https":[],
+          "https":[8443],
           "ws":[8888]},
  "check_subdomains":false,
- "bind_hostname":%(bind_hostname)s}
+ "bind_hostname":%(bind_hostname)s,
+ "ssl":{}}
--- a/testing/web-platform/harness/wptrunner/executors/executorservo.py
+++ b/testing/web-platform/harness/wptrunner/executors/executorservo.py
@@ -1,20 +1,25 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import hashlib
 import json
+import os
 import subprocess
+import tempfile
 import threading
 import urlparse
+import uuid
+from collections import defaultdict
 
 from mozprocess import ProcessHandler
 
-from .base import testharness_result_converter
+from .base import testharness_result_converter, reftest_result_converter
 from .process import ProcessTestExecutor
 
 
 class ServoTestharnessExecutor(ProcessTestExecutor):
     convert_result = testharness_result_converter
 
     def __init__(self, *args, **kwargs):
         ProcessTestExecutor.__init__(self, *args, **kwargs)
@@ -65,8 +70,99 @@ class ServoTestharnessExecutor(ProcessTe
                 print line
             else:
                 self.logger.process_output(self.proc.pid,
                                            line,
                                            " ".join(self.command))
 
     def on_finish(self):
         self.result_flag.set()
+
+
+class TempFilename(object):
+    def __init__(self, directory):
+        self.directory = directory
+        self.path = None
+
+    def __enter__(self):
+        self.path = os.path.join(self.directory, str(uuid.uuid4()))
+        return self.path
+
+    def __exit__(self, *args, **kwargs):
+        try:
+            os.unlink(self.path)
+        except OSError:
+            pass
+
+
+class ServoReftestExecutor(ProcessTestExecutor):
+    convert_result = reftest_result_converter
+
+    def __init__(self, *args, **kwargs):
+        ProcessTestExecutor.__init__(self, *args, **kwargs)
+        self.ref_hashes = {}
+        self.ref_urls_by_hash = defaultdict(set)
+        self.tempdir = tempfile.mkdtemp()
+
+    def teardown(self):
+        os.rmdir(self.tempdir)
+        ProcessTestExecutor.teardown(self)
+
+    def run_test(self, test):
+        test_url, ref_type, ref_url = test.url, test.ref_type, test.ref_url
+        hashes = {"test": None,
+                  "ref": self.ref_hashes.get(ref_url)}
+
+        status = None
+
+        for url_type, url in [("test", test_url), ("ref", ref_url)]:
+            if hashes[url_type] is None:
+                full_url = urlparse.urljoin(self.http_server_url, url)
+
+                with TempFilename(self.tempdir) as output_path:
+                    self.command = [self.binary, "--cpu", "--hard-fail", "--exit",
+                                    "--output=%s" % output_path, full_url]
+
+                    timeout = test.timeout * self.timeout_multiplier
+                    self.proc = ProcessHandler(self.command,
+                                               processOutputLine=[self.on_output])
+                    self.proc.run()
+                    rv = self.proc.wait(timeout=timeout)
+
+                    if rv is None:
+                        status = "EXTERNAL-TIMEOUT"
+                        self.proc.kill()
+                        break
+
+                    if rv < 0:
+                        status = "CRASH"
+                        break
+
+                    with open(output_path) as f:
+                        # Might need to strip variable headers or something here
+                        data = f.read()
+                        hashes[url_type] = hashlib.sha1(data).hexdigest()
+
+        if status is None:
+            self.ref_urls_by_hash[hashes["ref"]].add(ref_url)
+            self.ref_hashes[ref_url] = hashes["ref"]
+
+            if ref_type == "==":
+                passed = hashes["test"] == hashes["ref"]
+            elif ref_type == "!=":
+                passed = hashes["test"] != hashes["ref"]
+            else:
+                raise ValueError
+
+            status = "PASS" if passed else "FAIL"
+
+        result = self.convert_result(test, {"status": status, "message": None})
+        self.runner.send_message("test_ended", test, result)
+
+
+    def on_output(self, line):
+        line = line.decode("utf8", "replace")
+        if self.interactive:
+            print line
+        else:
+            self.logger.process_output(self.proc.pid,
+                                       line,
+                                       " ".join(self.command))
--- a/testing/web-platform/harness/wptrunner/products.py
+++ b/testing/web-platform/harness/wptrunner/products.py
@@ -44,9 +44,12 @@ def load_product(config, product):
     executor_kwargs = getattr(module, data["executor_kwargs"])
     env_options = getattr(module, data["env_options"])()
 
     executor_classes = {}
     for test_type, cls_name in data["executor"].iteritems():
         cls = getattr(module, cls_name)
         executor_classes[test_type] = cls
 
-    return check_args, browser_cls, browser_kwargs, executor_classes, executor_kwargs, env_options
+    return (check_args,
+            browser_cls, browser_kwargs,
+            executor_classes, executor_kwargs,
+            env_options)
--- a/testing/web-platform/harness/wptrunner/wptcommandline.py
+++ b/testing/web-platform/harness/wptrunner/wptcommandline.py
@@ -2,16 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import argparse
 import ast
 import os
 import sys
 from collections import OrderedDict
+from distutils.spawn import find_executable
 
 import config
 
 
 def abs_path(path):
     return os.path.abspath(os.path.expanduser(path))
 
 
@@ -119,16 +120,34 @@ def create_parser(product_choices=None):
     parser.add_argument('--pause-on-unexpected', action="store_true",
                         help="Halt the test runner when an unexpected result is encountered")
 
     parser.add_argument("--symbols-path", action="store", type=url_or_path,
                         help="Path or url to symbols file used to analyse crash minidumps.")
     parser.add_argument("--stackwalk-binary", action="store", type=abs_path,
                         help="Path to stackwalker program used to analyse minidumps.")
 
+    parser.add_argument("--ssl-type", action="store", default=None,
+                        choices=["openssl", "pregenerated", "none"],
+                        help="Type of ssl support to enable (running without ssl may lead to spurious errors)")
+
+    parser.add_argument("--openssl-binary", action="store",
+                        help="Path to openssl binary", default="openssl")
+    parser.add_argument("--certutil-binary", action="store",
+                        help="Path to certutil binary for use with Firefox + ssl")
+
+
+    parser.add_argument("--ca-cert-path", action="store", type=abs_path,
+                        help="Path to ca certificate when using pregenerated ssl certificates")
+    parser.add_argument("--host-key-path", action="store", type=abs_path,
+                        help="Path to host private key when using pregenerated ssl certificates")
+    parser.add_argument("--host-cert-path", action="store", type=abs_path,
+                        help="Path to host certificate when using pregenerated ssl certificates")
+
+
     parser.add_argument("--b2g-no-backup", action="store_true", default=False,
                         help="Don't backup device before testrun with --product=b2g")
 
     commandline.add_logging_group(parser)
     return parser
 
 
 def set_from_config(kwargs):
@@ -140,25 +159,30 @@ def set_from_config(kwargs):
     kwargs["config_path"] = config_path
     kwargs["config"] = config.read(kwargs["config_path"])
 
     keys = {"paths": [("serve", "serve_root", True),
                       ("prefs", "prefs_root", True),
                       ("run_info", "run_info", True)],
             "web-platform-tests": [("remote_url", "remote_url", False),
                                    ("branch", "branch", False),
-                                   ("sync_path", "sync_path", True)]}
+                                   ("sync_path", "sync_path", True)],
+            "SSL": [("openssl_binary", "openssl_binary", True),
+                    ("certutil_binary", "certutil_binary", True),
+                    ("ca_cert_path", "ca_cert_path", True),
+                    ("host_cert_path", "host_cert_path", True),
+                    ("host_key_path", "host_key_path", True)]}
 
     for section, values in keys.iteritems():
         for config_value, kw_value, is_path in values:
             if kw_value in kwargs and kwargs[kw_value] is None:
                 if not is_path:
-                    new_value = kwargs["config"].get(section, {}).get(config_value)
+                    new_value = kwargs["config"].get(section, config.ConfigDict({})).get(config_value)
                 else:
-                    new_value = kwargs["config"].get(section, {}).get_path(config_value)
+                    new_value = kwargs["config"].get(section, config.ConfigDict({})).get_path(config_value)
                 kwargs[kw_value] = new_value
 
     kwargs["test_paths"] = get_test_paths(kwargs["config"])
 
     if kwargs["tests_root"]:
         if "/" not in kwargs["test_paths"]:
             kwargs["test_paths"]["/"] = {}
         kwargs["test_paths"]["/"]["tests_path"] = kwargs["tests_root"]
@@ -179,16 +203,21 @@ def get_test_paths(config):
             test_paths[url_base] = {
                 "tests_path": manifest_opts.get_path("tests"),
                 "metadata_path": manifest_opts.get_path("metadata")}
 
     return test_paths
 
 
 
+def exe_path(name):
+    path = find_executable(name)
+    if os.access(path, os.X_OK):
+        return path
+
 def check_args(kwargs):
     from mozrunner import debugger_arguments
 
     set_from_config(kwargs)
 
     for test_paths in kwargs["test_paths"].itervalues():
         if not ("tests_path" in test_paths and
                 "metadata_path" in test_paths):
@@ -236,16 +265,43 @@ def check_args(kwargs):
         kwargs["interactive"] = False
         kwargs["debug_args"] = None
 
     if kwargs["binary"] is not None:
         if not os.path.exists(kwargs["binary"]):
             print >> sys.stderr, "Binary path %s does not exist" % kwargs["binary"]
             sys.exit(1)
 
+    if kwargs["ssl_type"] is None:
+        if None not in (kwargs["ca_cert_path"], kwargs["host_cert_path"], kwargs["host_key_path"]):
+            kwargs["ssl_type"] = "pregenerated"
+        elif exe_path(kwargs["openssl_binary"]) is not None:
+            kwargs["ssl_type"] = "openssl"
+        else:
+            kwargs["ssl_type"] = "none"
+
+    if kwargs["ssl_type"] == "pregenerated":
+        require_arg(kwargs, "ca_cert_path", lambda x:os.path.exists(x))
+        require_arg(kwargs, "host_cert_path", lambda x:os.path.exists(x))
+        require_arg(kwargs, "host_key_path", lambda x:os.path.exists(x))
+
+    elif kwargs["ssl_type"] == "openssl":
+        path = exe_path(kwargs["openssl_binary"])
+        if path is None:
+            print >> sys.stderr, "openssl-binary argument missing or not a valid executable"
+            sys.exit(1)
+        kwargs["openssl_binary"] = path
+
+    if kwargs["ssl_type"] != "none" and kwargs["product"] == "firefox":
+        path = exe_path(kwargs["certutil_binary"])
+        if path is None:
+            print >> sys.stderr, "certutil-binary argument missing or not a valid executable"
+            sys.exit(1)
+        kwargs["certutil_binary"] = path
+
     return kwargs
 
 
 def create_parser_update():
     parser = argparse.ArgumentParser("web-platform-tests-update",
                                      description="Update script for web-platform-tests tests.")
     parser.add_argument("--config", action="store", type=abs_path, help="Path to config file")
     parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root",
--- a/testing/web-platform/harness/wptrunner/wptrunner.py
+++ b/testing/web-platform/harness/wptrunner/wptrunner.py
@@ -39,17 +39,16 @@ The runner has several design goals:
 * Tests should be regarded as "untrusted" so that errors, timeouts and even
   crashes in the tests can be handled without failing the entire test run.
 
 * For performance tests can be run in multiple browsers in parallel.
 
 The upstream repository has the facility for creating a test manifest in JSON
 format. This manifest is used directly to determine which tests exist. Local
 metadata files are used to store the expected test results.
-
 """
 
 logger = None
 
 
 def setup_logging(args, defaults):
     global logger
     logger = commandline.setup_logging("web-platform-tests", args, defaults)
@@ -63,34 +62,41 @@ def setup_logging(args, defaults):
 
 
 def setup_stdlib_logger():
     logging.root.handlers = []
     logging.root = stdadapter.std_logging_adapter(logging.root)
 
 
 def do_delayed_imports(serve_root):
-    global serve, manifest
+    global serve, manifest, sslutils
 
-    sys.path.insert(0, os.path.join(serve_root))
-    sys.path.insert(0, os.path.join(serve_root, "tools", "scripts"))
-    failed = None
+    sys.path.insert(0, serve_root)
+    sys.path.insert(0, str(os.path.join(serve_root, "tools")))
+    sys.path.insert(0, str(os.path.join(serve_root, "tools", "scripts")))
+    failed = []
+
     try:
         import serve
     except ImportError:
-        failed = "serve"
+        failed.append("serve")
     try:
         import manifest
     except ImportError:
-        failed = "manifest"
+        failed.append("manifest")
+    try:
+        import sslutils
+    except ImportError:
+        raise
+        failed.append("sslutils")
 
     if failed:
         logger.critical(
             "Failed to import %s. Ensure that tests path %s contains web-platform-tests" %
-            (failed, serve_root))
+            (", ".join(failed), serve_root))
         sys.exit(1)
 
 
 class TestEnvironmentError(Exception):
     pass
 
 
 class LogLevelRewriter(object):
@@ -112,60 +118,78 @@ class LogLevelRewriter(object):
     def __call__(self, data):
         if data["action"] == "log" and data["level"].upper() in self.from_levels:
             data = data.copy()
             data["level"] = self.to_level
         return self.inner(data)
 
 
 class TestEnvironment(object):
-    def __init__(self, serve_path, test_paths, options):
+    def __init__(self, serve_path, test_paths, ssl_env, options):
         """Context manager that owns the test environment i.e. the http and
         websockets servers"""
         self.serve_path = serve_path
         self.test_paths = test_paths
+        self.ssl_env = ssl_env
         self.server = None
         self.config = None
         self.external_config = None
         self.test_server_port = options.pop("test_server_port", True)
         self.options = options if options is not None else {}
         self.required_files = options.pop("required_files", [])
         self.files_to_restore = []
 
     def __enter__(self):
+        self.ssl_env.__enter__()
         self.copy_required_files()
         self.setup_server_logging()
         self.setup_routes()
         self.config = self.load_config()
         serve.set_computed_defaults(self.config)
-        self.external_config, self.servers = serve.start(self.config)
+        self.external_config, self.servers = serve.start(self.config, self.ssl_env)
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
+        self.ssl_env.__exit__(exc_type, exc_val, exc_tb)
+
         self.restore_files()
         for scheme, servers in self.servers.iteritems():
             for port, server in servers:
                 server.kill()
 
     def load_config(self):
         default_config_path = os.path.join(self.serve_path, "config.default.json")
         local_config_path = os.path.join(here, "config.json")
 
         with open(default_config_path) as f:
             default_config = json.load(f)
 
         with open(local_config_path) as f:
             data = f.read()
             local_config = json.loads(data % self.options)
 
+        #TODO: allow non-default configuration for ssl
+
         local_config["external_host"] = self.options.get("external_host", None)
+        local_config["ssl"]["encrypt_after_connect"] = self.options.get("encrypt_after_connect", False)
 
         config = serve.merge_json(default_config, local_config)
         config["doc_root"] = self.serve_path
 
+        if not self.ssl_env.ssl_enabled:
+            config["ports"]["https"] = [None]
+
+        host = self.options.get("certificate_domain", config["host"])
+        hosts = [host]
+        hosts.extend("%s.%s" % (item[0], host) for item in serve.get_subdomains(host).values())
+        key_file, certificate = self.ssl_env.host_cert_path(hosts)
+
+        config["key_file"] = key_file
+        config["certificate"] = certificate
+
         return config
 
     def setup_server_logging(self):
         server_logger = get_default_logger(component="wptserve")
         assert server_logger is not None
         log_filter = handlers.LogLevelFilter(lambda x:x, "info")
         # Downgrade errors to warnings for the server
         log_filter = LogLevelRewriter(log_filter, ["error"], "warning")
@@ -295,31 +319,44 @@ def list_test_groups(serve_root, test_pa
     test_loader = testloader.TestLoader(test_paths,
                                         test_types,
                                         test_filter,
                                         run_info)
 
     for item in sorted(test_loader.groups(test_types)):
         print item
 
+
 def list_disabled(serve_root, test_paths, test_types, product, **kwargs):
     do_delayed_imports(serve_root)
     rv = []
     run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False)
     test_loader = testloader.TestLoader(test_paths,
                                         test_types,
                                         testloader.TestFilter(),
                                         run_info)
 
     for test_type, tests in test_loader.disabled_tests.iteritems():
         for test in tests:
             rv.append({"test": test.id, "reason": test.disabled()})
     print json.dumps(rv, indent=2)
 
 
+def get_ssl_kwargs(**kwargs):
+    if kwargs["ssl_type"] == "openssl":
+        args = {"openssl_binary": kwargs["openssl_binary"]}
+    elif kwargs["ssl_type"] == "pregenerated":
+        args = {"host_key_path": kwargs["host_key_path"],
+                "host_cert_path": kwargs["host_cert_path"],
+                 "ca_cert_path": kwargs["ca_cert_path"]}
+    else:
+        args = {}
+    return args
+
+
 def run_tests(config, serve_root, test_paths, product, **kwargs):
     logging_queue = None
     logging_thread = None
     original_stdio = (sys.stdout, sys.stderr)
     test_queues = None
 
     try:
         if not kwargs["no_capture_stdio"]:
@@ -335,17 +372,18 @@ def run_tests(config, serve_root, test_p
 
         (check_args,
          browser_cls, get_browser_kwargs,
          executor_classes, get_executor_kwargs,
          env_options) = products.load_product(config, product)
 
         check_args(**kwargs)
 
-        browser_kwargs = get_browser_kwargs(**kwargs)
+        ssl_env_cls = sslutils.environments[kwargs["ssl_type"]]
+        ssl_env = ssl_env_cls(logger, **get_ssl_kwargs(**kwargs))
 
         unexpected_total = 0
 
         if "test_loader" in kwargs:
             test_loader = kwargs["test_loader"]
         else:
             test_filter = testloader.TestFilter(include=kwargs["include"],
                                                 exclude=kwargs["exclude"],
@@ -366,25 +404,28 @@ def run_tests(config, serve_root, test_p
             # A value of None indicates infinite depth
             test_source_cls = testloader.PathGroupedSource
             test_source_kwargs = {"depth": kwargs["run_by_dir"]}
 
         logger.info("Using %i client processes" % kwargs["processes"])
 
         with TestEnvironment(serve_root,
                              test_paths,
+                             ssl_env,
                              env_options) as test_environment:
             try:
                 test_environment.ensure_started()
             except TestEnvironmentError as e:
                 logger.critical("Error starting test environment: %s" % e.message)
                 raise
 
+            browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs)
             base_server = "http://%s:%i" % (test_environment.external_config["host"],
                                             test_environment.external_config["ports"]["http"][0])
+
             repeat = kwargs["repeat"]
             for repeat_count in xrange(repeat):
                 if repeat > 1:
                     logger.info("Repetition %i / %i" % (repeat_count + 1, repeat))
 
 
                 unexpected_count = 0
                 logger.suite_start(test_loader.test_ids, run_info)