Bug 1025066 - Update to ssl-supporting wptrunner, a=testonly
authorJames Graham <james@hoppipolla.co.uk>
Sat, 10 Jan 2015 10:42:43 +0000
changeset 223173 47671b5f66ab8182bebfe255f946b500cab65908
parent 223172 9de247dc316174bd7c35d1e7449a82a5318cc153
child 223174 d16c7eac45de738cfbee6c902ea41a87232232e7
push id10769
push usercbook@mozilla.com
push dateMon, 12 Jan 2015 14:15:52 +0000
treeherderfx-team@0e9765732906 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1025066
milestone37.0a1
Bug 1025066 - Update to ssl-supporting wptrunner, a=testonly
testing/web-platform/harness/requirements.txt
testing/web-platform/harness/setup.py
testing/web-platform/harness/wptrunner/browsers/b2g.py
testing/web-platform/harness/wptrunner/browsers/base.py
testing/web-platform/harness/wptrunner/browsers/chrome.py
testing/web-platform/harness/wptrunner/browsers/firefox.py
testing/web-platform/harness/wptrunner/browsers/server-locations.txt
testing/web-platform/harness/wptrunner/browsers/servo.py
testing/web-platform/harness/wptrunner/config.json
testing/web-platform/harness/wptrunner/executors/executorservo.py
testing/web-platform/harness/wptrunner/products.py
testing/web-platform/harness/wptrunner/wptcommandline.py
testing/web-platform/harness/wptrunner/wptrunner.py
--- a/testing/web-platform/harness/requirements.txt
+++ b/testing/web-platform/harness/requirements.txt
@@ -1,5 +1,5 @@
 html5lib >= 0.99
 mozinfo >= 0.7
-mozlog >= 1.8
+mozlog >= 2.8
 # Unfortunately, just for gdb flags
 mozrunner >= 6.1
--- a/testing/web-platform/harness/setup.py
+++ b/testing/web-platform/harness/setup.py
@@ -7,17 +7,17 @@ import os
 import sys
 import textwrap
 
 from setuptools import setup, find_packages
 
 here = os.path.split(__file__)[0]
 
 PACKAGE_NAME = 'wptrunner'
-PACKAGE_VERSION = '1.7'
+PACKAGE_VERSION = '1.8'
 
 # Dependencies
 with open(os.path.join(here, "requirements.txt")) as f:
     deps = f.read().splitlines()
 
 # Browser-specific requirements
 requirements_files = glob.glob(os.path.join(here, "requirements_*.txt"))
 
--- a/testing/web-platform/harness/wptrunner/browsers/b2g.py
+++ b/testing/web-platform/harness/wptrunner/browsers/b2g.py
@@ -31,17 +31,17 @@ here = os.path.split(__file__)[0]
                  "executor_kwargs": "executor_kwargs",
                  "env_options": "env_options"}
 
 
 def check_args(**kwargs):
     pass
 
 
-def browser_kwargs(**kwargs):
+def browser_kwargs(test_environment, **kwargs):
     return {"prefs_root": kwargs["prefs_root"],
             "no_backup": kwargs.get("b2g_no_backup", False)}
 
 
 def executor_kwargs(http_server_url, **kwargs):
     timeout_multiplier = kwargs["timeout_multiplier"]
     if timeout_multiplier is None:
         timeout_multiplier = 2
--- a/testing/web-platform/harness/wptrunner/browsers/base.py
+++ b/testing/web-platform/harness/wptrunner/browsers/base.py
@@ -90,16 +90,20 @@ class Browser(object):
         """pid of the browser process or None if there is no pid"""
         pass
 
     @abstractmethod
     def is_alive(self):
         """Boolean indicating whether the browser process is still running"""
         pass
 
+    def setup_ssl(self, hosts):
+        """Return a certificate to use for tests requiring ssl that will be trusted by the browser"""
+        raise NotImplementedError("ssl testing not supported")
+
     def cleanup(self):
         """Browser-specific cleanup that is run after the testrun is finished"""
         pass
 
     def executor_browser(self):
         """Returns the ExecutorBrowser subclass for this Browser subclass and the keyword arguments
         with which it should be instantiated"""
         return ExecutorBrowser, {}
--- a/testing/web-platform/harness/wptrunner/browsers/chrome.py
+++ b/testing/web-platform/harness/wptrunner/browsers/chrome.py
@@ -36,17 +36,17 @@ def executor_kwargs(http_server_url, **k
                         {"chromeOptions": {"binary": binary}}.items())
 
     return {"http_server_url": http_server_url,
             "capabilities": capabilities,
             "timeout_multiplier": timeout_multiplier}
 
 
 def env_options():
-    return {"host": "localhost",
+    return {"host": "web-platform.test",
             "bind_hostname": "true",
             "required_files": required_files}
 
 
 class ChromeBrowser(Browser):
     """Chrome is backed by chromedriver, which is supplied through
     ``browsers.webdriver.ChromedriverLocalServer``."""
 
--- a/testing/web-platform/harness/wptrunner/browsers/firefox.py
+++ b/testing/web-platform/harness/wptrunner/browsers/firefox.py
@@ -1,14 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
+import subprocess
 
+import mozinfo
 from mozprocess import ProcessHandler
 from mozprofile import FirefoxProfile, Preferences
 from mozprofile.permissions import ServerLocations
 from mozrunner import FirefoxRunner
 from mozcrash import mozcrash
 
 from .base import get_free_port, Browser, ExecutorBrowser, require_arg, cmd_arg
 from ..executors import executor_kwargs as base_executor_kwargs
@@ -23,56 +25,65 @@ here = os.path.join(os.path.split(__file
                               "reftest": "MarionetteReftestExecutor"},
                  "browser_kwargs": "browser_kwargs",
                  "executor_kwargs": "executor_kwargs",
                  "env_options": "env_options"}
 
 
 def check_args(**kwargs):
     require_arg(kwargs, "binary")
+    if kwargs["ssl_type"] != "none":
+        require_arg(kwargs, "certutil_binary")
 
 
 def browser_kwargs(**kwargs):
     return {"binary": kwargs["binary"],
             "prefs_root": kwargs["prefs_root"],
             "debug_args": kwargs["debug_args"],
             "interactive": kwargs["interactive"],
-            "symbols_path":kwargs["symbols_path"],
-            "stackwalk_binary":kwargs["stackwalk_binary"]}
+            "symbols_path": kwargs["symbols_path"],
+            "stackwalk_binary": kwargs["stackwalk_binary"],
+            "certutil_binary": kwargs["certutil_binary"],
+            "ca_certificate_path": kwargs["ssl_env"].ca_cert_path()}
 
 
 def executor_kwargs(http_server_url, **kwargs):
     executor_kwargs = base_executor_kwargs(http_server_url, **kwargs)
     executor_kwargs["close_after_done"] = True
     return executor_kwargs
 
 
 def env_options():
-    return {"host": "localhost",
+    return {"host": "127.0.0.1",
             "external_host": "web-platform.test",
-            "bind_hostname": "true",
-            "required_files": required_files}
+            "bind_hostname": "false",
+            "required_files": required_files,
+            "certificate_domain": "web-platform.test",
+            "encrypt_after_connect": True}
 
 
 class FirefoxBrowser(Browser):
     used_ports = set()
 
     def __init__(self, logger, binary, prefs_root, debug_args=None, interactive=None,
-                 symbols_path=None, stackwalk_binary=None):
+                 symbols_path=None, stackwalk_binary=None, certutil_binary=None,
+                 ca_certificate_path=None):
         Browser.__init__(self, logger)
         self.binary = binary
         self.prefs_root = prefs_root
         self.marionette_port = None
         self.used_ports.add(self.marionette_port)
         self.runner = None
         self.debug_args = debug_args
         self.interactive = interactive
         self.profile = None
         self.symbols_path = symbols_path
         self.stackwalk_binary = stackwalk_binary
+        self.ca_certificate_path = ca_certificate_path
+        self.certutil_binary = certutil_binary
 
     def start(self):
         self.marionette_port = get_free_port(2828, exclude=self.used_ports)
 
         env = os.environ.copy()
         env["MOZ_CRASHREPORTER"] = "1"
         env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
         env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
@@ -88,16 +99,19 @@ class FirefoxBrowser(Browser):
 
         self.profile = FirefoxProfile(locations=locations,
                                       proxy=ports,
                                       preferences=preferences)
         self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
                                       "marionette.defaultPrefs.port": self.marionette_port,
                                       "dom.disable_open_during_load": False})
 
+        if self.ca_certificate_path is not None:
+            self.setup_ssl()
+
         self.runner = FirefoxRunner(profile=self.profile,
                                     binary=self.binary,
                                     cmdargs=[cmd_arg("marionette"), "about:blank"],
                                     env=env,
                                     process_class=ProcessHandler,
                                     process_args={"processOutputLine": [self.on_output]})
 
         self.logger.debug("Starting Firefox")
@@ -147,11 +161,55 @@ class FirefoxBrowser(Browser):
         self.stop()
 
     def executor_browser(self):
         assert self.marionette_port is not None
         return ExecutorBrowser, {"marionette_port": self.marionette_port}
 
     def log_crash(self, process, test):
         dump_dir = os.path.join(self.profile.profile, "minidumps")
-        mozcrash.log_crashes(self.logger, dump_dir, symbols_path=self.symbols_path,
+
+        mozcrash.log_crashes(self.logger,
+                             dump_dir,
+                             symbols_path=self.symbols_path,
                              stackwalk_binary=self.stackwalk_binary,
-                             process=process, test=test)
+                             process=process,
+                             test=test)
+
+    def setup_ssl(self):
+        """Create a certificate database to use in the test profile. This is configured
+        to trust the CA Certificate that has signed the web-platform.test server
+        certificate."""
+
+        self.logger.info("Setting up ssl")
+
+        # Make sure the certutil libraries from the source tree are loaded when using a
+        # local copy of certutil
+        # TODO: Maybe only set this if certutil won't launch?
+        env = os.environ.copy()
+        certutil_dir = os.path.dirname(self.binary)
+        env["LD_LIBRARY_PATH"] = certutil_dir
+        env["PATH"] = os.path.pathsep.join([certutil_dir, env["PATH"]])
+
+        def certutil(*args):
+            cmd = [self.certutil_binary] + list(args)
+            self.logger.process_output("certutil",
+                                       subprocess.check_output(cmd,
+                                                               env=env,
+                                                               stderr=subprocess.STDOUT),
+                                       " ".join(cmd))
+
+        pw_path = os.path.join(self.profile.profile, ".crtdbpw")
+        with open(pw_path, "w") as f:
+            # Use empty password for certificate db
+            f.write("\n")
+
+        cert_db_path = self.profile.profile
+
+        # Create a new certificate db
+        certutil("-N", "-d", cert_db_path, "-f", pw_path)
+
+        # Add the CA certificate to the database and mark as trusted to issue server certs
+        certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,",
+                 "-n", "web-platform-tests", "-i", self.ca_certificate_path)
+
+        # List all certs in the database
+        certutil("-L", "-d", cert_db_path)
--- a/testing/web-platform/harness/wptrunner/browsers/server-locations.txt
+++ b/testing/web-platform/harness/wptrunner/browsers/server-locations.txt
@@ -16,16 +16,23 @@ http://xn--lve-6lad.web-platform.test:80
 
 http://web-platform.test:8001
 http://www.web-platform.test:8001
 http://www1.web-platform.test:8001
 http://www2.web-platform.test:8001
 http://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8001
 http://xn--lve-6lad.web-platform.test:8001
 
+https://web-platform.test:8443
+https://www.web-platform.test:8443
+https://www1.web-platform.test:8443
+https://www2.web-platform.test:8443
+https://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8443
+https://xn--lve-6lad.web-platform.test:8443
+
 # These are actually ws servers, but until mozprofile is
 # fixed we have to pretend that they are http servers
 http://web-platform.test:8888
 http://www.web-platform.test:8888
 http://www1.web-platform.test:8888
 http://www2.web-platform.test:8888
 http://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8888
 http://xn--lve-6lad.web-platform.test:8888
--- a/testing/web-platform/harness/wptrunner/browsers/servo.py
+++ b/testing/web-platform/harness/wptrunner/browsers/servo.py
@@ -1,24 +1,25 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
 
 from .base import NullBrowser, ExecutorBrowser, require_arg
 from ..executors import executor_kwargs
-from ..executors.executorservo import ServoTestharnessExecutor
+from ..executors.executorservo import ServoTestharnessExecutor, ServoReftestExecutor
 
 here = os.path.join(os.path.split(__file__)[0])
 
 __wptrunner__ = {"product": "servo",
                  "check_args": "check_args",
                  "browser": "ServoBrowser",
-                 "executor": {"testharness": "ServoTestharnessExecutor"},
+                 "executor": {"testharness": "ServoTestharnessExecutor",
+                              "reftest": "ServoReftestExecutor"},
                  "browser_kwargs": "browser_kwargs",
                  "executor_kwargs": "executor_kwargs",
                  "env_options": "env_options"}
 
 
 def check_args(**kwargs):
     require_arg(kwargs, "binary")
 
--- a/testing/web-platform/harness/wptrunner/config.json
+++ b/testing/web-platform/harness/wptrunner/config.json
@@ -1,6 +1,7 @@
 {"host": "%(host)s",
  "ports":{"http":[8000, 8001],
-          "https":[],
+          "https":[8443],
           "ws":[8888]},
  "check_subdomains":false,
- "bind_hostname":%(bind_hostname)s}
+ "bind_hostname":%(bind_hostname)s,
+ "ssl":{}}
--- a/testing/web-platform/harness/wptrunner/executors/executorservo.py
+++ b/testing/web-platform/harness/wptrunner/executors/executorservo.py
@@ -1,20 +1,25 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import hashlib
 import json
+import os
 import subprocess
+import tempfile
 import threading
 import urlparse
+import uuid
+from collections import defaultdict
 
 from mozprocess import ProcessHandler
 
-from .base import testharness_result_converter
+from .base import testharness_result_converter, reftest_result_converter
 from .process import ProcessTestExecutor
 
 
 class ServoTestharnessExecutor(ProcessTestExecutor):
     convert_result = testharness_result_converter
 
     def __init__(self, *args, **kwargs):
         ProcessTestExecutor.__init__(self, *args, **kwargs)
@@ -65,8 +70,99 @@ class ServoTestharnessExecutor(ProcessTe
                 print line
             else:
                 self.logger.process_output(self.proc.pid,
                                            line,
                                            " ".join(self.command))
 
     def on_finish(self):
         self.result_flag.set()
+
+
+class TempFilename(object):
+    def __init__(self, directory):
+        self.directory = directory
+        self.path = None
+
+    def __enter__(self):
+        self.path = os.path.join(self.directory, str(uuid.uuid4()))
+        return self.path
+
+    def __exit__(self, *args, **kwargs):
+        try:
+            os.unlink(self.path)
+        except OSError:
+            pass
+
+
+class ServoReftestExecutor(ProcessTestExecutor):
+    convert_result = reftest_result_converter
+
+    def __init__(self, *args, **kwargs):
+        ProcessTestExecutor.__init__(self, *args, **kwargs)
+        self.ref_hashes = {}
+        self.ref_urls_by_hash = defaultdict(set)
+        self.tempdir = tempfile.mkdtemp()
+
+    def teardown(self):
+        os.rmdir(self.tempdir)
+        ProcessTestExecutor.teardown(self)
+
+    def run_test(self, test):
+        test_url, ref_type, ref_url = test.url, test.ref_type, test.ref_url
+        hashes = {"test": None,
+                  "ref": self.ref_hashes.get(ref_url)}
+
+        status = None
+
+        for url_type, url in [("test", test_url), ("ref", ref_url)]:
+            if hashes[url_type] is None:
+                full_url = urlparse.urljoin(self.http_server_url, url)
+
+                with TempFilename(self.tempdir) as output_path:
+                    self.command = [self.binary, "--cpu", "--hard-fail", "--exit",
+                                    "--output=%s" % output_path, full_url]
+
+                    timeout = test.timeout * self.timeout_multiplier
+                    self.proc = ProcessHandler(self.command,
+                                               processOutputLine=[self.on_output])
+                    self.proc.run()
+                    rv = self.proc.wait(timeout=timeout)
+
+                    if rv is None:
+                        status = "EXTERNAL-TIMEOUT"
+                        self.proc.kill()
+                        break
+
+                    if rv < 0:
+                        status = "CRASH"
+                        break
+
+                    with open(output_path) as f:
+                        # Might need to strip variable headers or something here
+                        data = f.read()
+                        hashes[url_type] = hashlib.sha1(data).hexdigest()
+
+        if status is None:
+            self.ref_urls_by_hash[hashes["ref"]].add(ref_url)
+            self.ref_hashes[ref_url] = hashes["ref"]
+
+            if ref_type == "==":
+                passed = hashes["test"] == hashes["ref"]
+            elif ref_type == "!=":
+                passed = hashes["test"] != hashes["ref"]
+            else:
+                raise ValueError
+
+            status = "PASS" if passed else "FAIL"
+
+        result = self.convert_result(test, {"status": status, "message": None})
+        self.runner.send_message("test_ended", test, result)
+
+
+    def on_output(self, line):
+        line = line.decode("utf8", "replace")
+        if self.interactive:
+            print line
+        else:
+            self.logger.process_output(self.proc.pid,
+                                       line,
+                                       " ".join(self.command))
--- a/testing/web-platform/harness/wptrunner/products.py
+++ b/testing/web-platform/harness/wptrunner/products.py
@@ -44,9 +44,12 @@ def load_product(config, product):
     executor_kwargs = getattr(module, data["executor_kwargs"])
     env_options = getattr(module, data["env_options"])()
 
     executor_classes = {}
     for test_type, cls_name in data["executor"].iteritems():
         cls = getattr(module, cls_name)
         executor_classes[test_type] = cls
 
-    return check_args, browser_cls, browser_kwargs, executor_classes, executor_kwargs, env_options
+    return (check_args,
+            browser_cls, browser_kwargs,
+            executor_classes, executor_kwargs,
+            env_options)
--- a/testing/web-platform/harness/wptrunner/wptcommandline.py
+++ b/testing/web-platform/harness/wptrunner/wptcommandline.py
@@ -2,16 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import argparse
 import ast
 import os
 import sys
 from collections import OrderedDict
+from distutils.spawn import find_executable
 
 import config
 
 
 def abs_path(path):
     return os.path.abspath(os.path.expanduser(path))
 
 
@@ -119,16 +120,34 @@ def create_parser(product_choices=None):
     parser.add_argument('--pause-on-unexpected', action="store_true",
                         help="Halt the test runner when an unexpected result is encountered")
 
     parser.add_argument("--symbols-path", action="store", type=url_or_path,
                         help="Path or url to symbols file used to analyse crash minidumps.")
     parser.add_argument("--stackwalk-binary", action="store", type=abs_path,
                         help="Path to stackwalker program used to analyse minidumps.")
 
+    parser.add_argument("--ssl-type", action="store", default=None,
+                        choices=["openssl", "pregenerated", "none"],
+                        help="Type of ssl support to enable (running without ssl may lead to spurious errors)")
+
+    parser.add_argument("--openssl-binary", action="store",
+                        help="Path to openssl binary", default="openssl")
+    parser.add_argument("--certutil-binary", action="store",
+                        help="Path to certutil binary for use with Firefox + ssl")
+
+
+    parser.add_argument("--ca-cert-path", action="store", type=abs_path,
+                        help="Path to ca certificate when using pregenerated ssl certificates")
+    parser.add_argument("--host-key-path", action="store", type=abs_path,
+                        help="Path to host private key when using pregenerated ssl certificates")
+    parser.add_argument("--host-cert-path", action="store", type=abs_path,
+                        help="Path to host certificate when using pregenerated ssl certificates")
+
+
     parser.add_argument("--b2g-no-backup", action="store_true", default=False,
                         help="Don't backup device before testrun with --product=b2g")
 
     commandline.add_logging_group(parser)
     return parser
 
 
 def set_from_config(kwargs):
@@ -140,25 +159,30 @@ def set_from_config(kwargs):
     kwargs["config_path"] = config_path
     kwargs["config"] = config.read(kwargs["config_path"])
 
     keys = {"paths": [("serve", "serve_root", True),
                       ("prefs", "prefs_root", True),
                       ("run_info", "run_info", True)],
             "web-platform-tests": [("remote_url", "remote_url", False),
                                    ("branch", "branch", False),
-                                   ("sync_path", "sync_path", True)]}
+                                   ("sync_path", "sync_path", True)],
+            "SSL": [("openssl_binary", "openssl_binary", True),
+                    ("certutil_binary", "certutil_binary", True),
+                    ("ca_cert_path", "ca_cert_path", True),
+                    ("host_cert_path", "host_cert_path", True),
+                    ("host_key_path", "host_key_path", True)]}
 
     for section, values in keys.iteritems():
         for config_value, kw_value, is_path in values:
             if kw_value in kwargs and kwargs[kw_value] is None:
                 if not is_path:
-                    new_value = kwargs["config"].get(section, {}).get(config_value)
+                    new_value = kwargs["config"].get(section, config.ConfigDict({})).get(config_value)
                 else:
-                    new_value = kwargs["config"].get(section, {}).get_path(config_value)
+                    new_value = kwargs["config"].get(section, config.ConfigDict({})).get_path(config_value)
                 kwargs[kw_value] = new_value
 
     kwargs["test_paths"] = get_test_paths(kwargs["config"])
 
     if kwargs["tests_root"]:
         if "/" not in kwargs["test_paths"]:
             kwargs["test_paths"]["/"] = {}
         kwargs["test_paths"]["/"]["tests_path"] = kwargs["tests_root"]
@@ -179,16 +203,21 @@ def get_test_paths(config):
             test_paths[url_base] = {
                 "tests_path": manifest_opts.get_path("tests"),
                 "metadata_path": manifest_opts.get_path("metadata")}
 
     return test_paths
 
 
 
+def exe_path(name):
+    path = find_executable(name)
+    if os.access(path, os.X_OK):
+        return path
+
 def check_args(kwargs):
     from mozrunner import debugger_arguments
 
     set_from_config(kwargs)
 
     for test_paths in kwargs["test_paths"].itervalues():
         if not ("tests_path" in test_paths and
                 "metadata_path" in test_paths):
@@ -236,16 +265,43 @@ def check_args(kwargs):
         kwargs["interactive"] = False
         kwargs["debug_args"] = None
 
     if kwargs["binary"] is not None:
         if not os.path.exists(kwargs["binary"]):
             print >> sys.stderr, "Binary path %s does not exist" % kwargs["binary"]
             sys.exit(1)
 
+    if kwargs["ssl_type"] is None:
+        if None not in (kwargs["ca_cert_path"], kwargs["host_cert_path"], kwargs["host_key_path"]):
+            kwargs["ssl_type"] = "pregenerated"
+        elif exe_path(kwargs["openssl_binary"]) is not None:
+            kwargs["ssl_type"] = "openssl"
+        else:
+            kwargs["ssl_type"] = "none"
+
+    if kwargs["ssl_type"] == "pregenerated":
+        require_arg(kwargs, "ca_cert_path", lambda x:os.path.exists(x))
+        require_arg(kwargs, "host_cert_path", lambda x:os.path.exists(x))
+        require_arg(kwargs, "host_key_path", lambda x:os.path.exists(x))
+
+    elif kwargs["ssl_type"] == "openssl":
+        path = exe_path(kwargs["openssl_binary"])
+        if path is None:
+            print >> sys.stderr, "openssl-binary argument missing or not a valid executable"
+            sys.exit(1)
+        kwargs["openssl_binary"] = path
+
+    if kwargs["ssl_type"] != "none" and kwargs["product"] == "firefox":
+        path = exe_path(kwargs["certutil_binary"])
+        if path is None:
+            print >> sys.stderr, "certutil-binary argument missing or not a valid executable"
+            sys.exit(1)
+        kwargs["certutil_binary"] = path
+
     return kwargs
 
 
 def create_parser_update():
     parser = argparse.ArgumentParser("web-platform-tests-update",
                                      description="Update script for web-platform-tests tests.")
     parser.add_argument("--config", action="store", type=abs_path, help="Path to config file")
     parser.add_argument("--metadata", action="store", type=abs_path, dest="metadata_root",
--- a/testing/web-platform/harness/wptrunner/wptrunner.py
+++ b/testing/web-platform/harness/wptrunner/wptrunner.py
@@ -39,17 +39,16 @@ The runner has several design goals:
 * Tests should be regarded as "untrusted" so that errors, timeouts and even
   crashes in the tests can be handled without failing the entire test run.
 
 * For performance tests can be run in multiple browsers in parallel.
 
 The upstream repository has the facility for creating a test manifest in JSON
 format. This manifest is used directly to determine which tests exist. Local
 metadata files are used to store the expected test results.
-
 """
 
 logger = None
 
 
 def setup_logging(args, defaults):
     global logger
     logger = commandline.setup_logging("web-platform-tests", args, defaults)
@@ -63,34 +62,41 @@ def setup_logging(args, defaults):
 
 
 def setup_stdlib_logger():
     logging.root.handlers = []
     logging.root = stdadapter.std_logging_adapter(logging.root)
 
 
 def do_delayed_imports(serve_root):
-    global serve, manifest
+    global serve, manifest, sslutils
 
-    sys.path.insert(0, os.path.join(serve_root))
-    sys.path.insert(0, os.path.join(serve_root, "tools", "scripts"))
-    failed = None
+    sys.path.insert(0, serve_root)
+    sys.path.insert(0, str(os.path.join(serve_root, "tools")))
+    sys.path.insert(0, str(os.path.join(serve_root, "tools", "scripts")))
+    failed = []
+
     try:
         import serve
     except ImportError:
-        failed = "serve"
+        failed.append("serve")
     try:
         import manifest
     except ImportError:
-        failed = "manifest"
+        failed.append("manifest")
+    try:
+        import sslutils
+    except ImportError:
+        raise
+        failed.append("sslutils")
 
     if failed:
         logger.critical(
             "Failed to import %s. Ensure that tests path %s contains web-platform-tests" %
-            (failed, serve_root))
+            (", ".join(failed), serve_root))
         sys.exit(1)
 
 
 class TestEnvironmentError(Exception):
     pass
 
 
 class LogLevelRewriter(object):
@@ -112,60 +118,78 @@ class LogLevelRewriter(object):
     def __call__(self, data):
         if data["action"] == "log" and data["level"].upper() in self.from_levels:
             data = data.copy()
             data["level"] = self.to_level
         return self.inner(data)
 
 
 class TestEnvironment(object):
-    def __init__(self, serve_path, test_paths, options):
+    def __init__(self, serve_path, test_paths, ssl_env, options):
         """Context manager that owns the test environment i.e. the http and
         websockets servers"""
         self.serve_path = serve_path
         self.test_paths = test_paths
+        self.ssl_env = ssl_env
         self.server = None
         self.config = None
         self.external_config = None
         self.test_server_port = options.pop("test_server_port", True)
         self.options = options if options is not None else {}
         self.required_files = options.pop("required_files", [])
         self.files_to_restore = []
 
     def __enter__(self):
+        self.ssl_env.__enter__()
         self.copy_required_files()
         self.setup_server_logging()
         self.setup_routes()
         self.config = self.load_config()
         serve.set_computed_defaults(self.config)
-        self.external_config, self.servers = serve.start(self.config)
+        self.external_config, self.servers = serve.start(self.config, self.ssl_env)
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
+        self.ssl_env.__exit__(exc_type, exc_val, exc_tb)
+
         self.restore_files()
         for scheme, servers in self.servers.iteritems():
             for port, server in servers:
                 server.kill()
 
     def load_config(self):
         default_config_path = os.path.join(self.serve_path, "config.default.json")
         local_config_path = os.path.join(here, "config.json")
 
         with open(default_config_path) as f:
             default_config = json.load(f)
 
         with open(local_config_path) as f:
             data = f.read()
             local_config = json.loads(data % self.options)
 
+        #TODO: allow non-default configuration for ssl
+
         local_config["external_host"] = self.options.get("external_host", None)
+        local_config["ssl"]["encrypt_after_connect"] = self.options.get("encrypt_after_connect", False)
 
         config = serve.merge_json(default_config, local_config)
         config["doc_root"] = self.serve_path
 
+        if not self.ssl_env.ssl_enabled:
+            config["ports"]["https"] = [None]
+
+        host = self.options.get("certificate_domain", config["host"])
+        hosts = [host]
+        hosts.extend("%s.%s" % (item[0], host) for item in serve.get_subdomains(host).values())
+        key_file, certificate = self.ssl_env.host_cert_path(hosts)
+
+        config["key_file"] = key_file
+        config["certificate"] = certificate
+
         return config
 
     def setup_server_logging(self):
         server_logger = get_default_logger(component="wptserve")
         assert server_logger is not None
         log_filter = handlers.LogLevelFilter(lambda x:x, "info")
         # Downgrade errors to warnings for the server
         log_filter = LogLevelRewriter(log_filter, ["error"], "warning")
@@ -295,31 +319,44 @@ def list_test_groups(serve_root, test_pa
     test_loader = testloader.TestLoader(test_paths,
                                         test_types,
                                         test_filter,
                                         run_info)
 
     for item in sorted(test_loader.groups(test_types)):
         print item
 
+
 def list_disabled(serve_root, test_paths, test_types, product, **kwargs):
     do_delayed_imports(serve_root)
     rv = []
     run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=False)
     test_loader = testloader.TestLoader(test_paths,
                                         test_types,
                                         testloader.TestFilter(),
                                         run_info)
 
     for test_type, tests in test_loader.disabled_tests.iteritems():
         for test in tests:
             rv.append({"test": test.id, "reason": test.disabled()})
     print json.dumps(rv, indent=2)
 
 
+def get_ssl_kwargs(**kwargs):
+    if kwargs["ssl_type"] == "openssl":
+        args = {"openssl_binary": kwargs["openssl_binary"]}
+    elif kwargs["ssl_type"] == "pregenerated":
+        args = {"host_key_path": kwargs["host_key_path"],
+                "host_cert_path": kwargs["host_cert_path"],
+                 "ca_cert_path": kwargs["ca_cert_path"]}
+    else:
+        args = {}
+    return args
+
+
 def run_tests(config, serve_root, test_paths, product, **kwargs):
     logging_queue = None
     logging_thread = None
     original_stdio = (sys.stdout, sys.stderr)
     test_queues = None
 
     try:
         if not kwargs["no_capture_stdio"]:
@@ -335,17 +372,18 @@ def run_tests(config, serve_root, test_p
 
         (check_args,
          browser_cls, get_browser_kwargs,
          executor_classes, get_executor_kwargs,
          env_options) = products.load_product(config, product)
 
         check_args(**kwargs)
 
-        browser_kwargs = get_browser_kwargs(**kwargs)
+        ssl_env_cls = sslutils.environments[kwargs["ssl_type"]]
+        ssl_env = ssl_env_cls(logger, **get_ssl_kwargs(**kwargs))
 
         unexpected_total = 0
 
         if "test_loader" in kwargs:
             test_loader = kwargs["test_loader"]
         else:
             test_filter = testloader.TestFilter(include=kwargs["include"],
                                                 exclude=kwargs["exclude"],
@@ -366,25 +404,28 @@ def run_tests(config, serve_root, test_p
             # A value of None indicates infinite depth
             test_source_cls = testloader.PathGroupedSource
             test_source_kwargs = {"depth": kwargs["run_by_dir"]}
 
         logger.info("Using %i client processes" % kwargs["processes"])
 
         with TestEnvironment(serve_root,
                              test_paths,
+                             ssl_env,
                              env_options) as test_environment:
             try:
                 test_environment.ensure_started()
             except TestEnvironmentError as e:
                 logger.critical("Error starting test environment: %s" % e.message)
                 raise
 
+            browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs)
             base_server = "http://%s:%i" % (test_environment.external_config["host"],
                                             test_environment.external_config["ports"]["http"][0])
+
             repeat = kwargs["repeat"]
             for repeat_count in xrange(repeat):
                 if repeat > 1:
                     logger.info("Repetition %i / %i" % (repeat_count + 1, repeat))
 
 
                 unexpected_count = 0
                 logger.suite_start(test_loader.test_ids, run_info)