Bug 1446463 [wpt PR 10078] - Get rid of last web-platform.test, make host_ip always present, a=testonly
authorGeoffrey Sneddon <me@gsnedders.com>
Mon, 09 Apr 2018 17:20:39 +0000
changeset 467114 2a008fb712f7a35307857434db2e7a89f63783a7
parent 467113 2a6220e4ae749024aeb3ff31de5544e4195d398f
child 467115 564cc5a6ee8eda293f5fe7b3e762137022f368ee
push id9165
push userasasaki@mozilla.com
push dateThu, 26 Apr 2018 21:04:54 +0000
treeherdermozilla-beta@064c3804de2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1446463, 10078
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1446463 [wpt PR 10078] - Get rid of last web-platform.test, make host_ip always present, a=testonly Automatic update from web-platform-testsMake host_ip always exist in the normalised config -- Get rid of redundant host/certificate_domain definitions -- Get rid of hardcoded web-platform.test in wpt/run.py -- Rename host -> browser_host and host_ip -> server_host wpt-commits: 66076b4a8a27fd6754f9a2155c17d607e4fad33f, 785ef2f057ca2b329a5604f234ce05209a1d8a75, be69acde3db9baca84efa29f087f7c6594e49925, f281dadb2890000c1afa222d071a4eb6c80d6963 wpt-pr: 10078 wpt-commits: 66076b4a8a27fd6754f9a2155c17d607e4fad33f, 785ef2f057ca2b329a5604f234ce05209a1d8a75, be69acde3db9baca84efa29f087f7c6594e49925, f281dadb2890000c1afa222d071a4eb6c80d6963 wpt-pr: 10078
testing/web-platform/meta/MANIFEST.json
testing/web-platform/tests/config.default.json
testing/web-platform/tests/tools/serve/serve.py
testing/web-platform/tests/tools/wpt/run.py
testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servo.py
testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servodriver.py
testing/web-platform/tests/tools/wptrunner/wptrunner/environment.py
testing/web-platform/tests/tools/wptrunner/wptrunner/executors/base.py
testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_products.py
testing/web-platform/tests/tools/wptserve/wptserve/pipes.py
testing/web-platform/tests/tools/wptserve/wptserve/server.py
--- a/testing/web-platform/meta/MANIFEST.json
+++ b/testing/web-platform/meta/MANIFEST.json
@@ -391402,17 +391402,17 @@
    "23780fdf7eea63c82965c8d5d4d847c0b9752c7e",
    "support"
   ],
   "./check_stability.ini": [
    "f8af4287f3b0f6925a2a6c5c75b3788e24de1680",
    "support"
   ],
   "./config.default.json": [
-   "939f06a73458155a62c9d81f198ac0194ad26234",
+   "54422ca469a8d54fa4b8252803bb0e2c91459858",
    "support"
   ],
   "./lint.whitelist": [
    "b35324fd499dcb7ba0a9deedc794bb13ee62e175",
    "support"
   ],
   "./serve.py": [
    "0efa39b1f26f86d73f2fce4f9e46003d62057b41",
--- a/testing/web-platform/tests/config.default.json
+++ b/testing/web-platform/tests/config.default.json
@@ -1,12 +1,12 @@
-{"host": "web-platform.test",
+{"browser_host": "web-platform.test",
  "doc_root": null,
  "ws_doc_root": null,
- "host_ip": null,
+ "server_host": null,
  "ports":{"http":[8000, "auto"],
           "https":[8443],
           "ws":["auto"],
           "wss":["auto"]},
  "check_subdomains": true,
  "log_level":"debug",
  "bind_address": true,
  "ssl": {"type": "pregenerated",
--- a/testing/web-platform/tests/tools/serve/serve.py
+++ b/testing/web-platform/tests/tools/serve/serve.py
@@ -641,17 +641,22 @@ def get_ports(config, ssl_environment):
             else:
                 port = port
             rv[scheme].append(port)
     return rv
 
 
 
 def normalise_config(config, ports):
-    host = config["host"]
+    if "host" in config:
+        logger.warning("host in config is deprecated; use browser_host instead")
+        host = config["host"]
+    else:
+        host = config["browser_host"]
+
     domains = get_subdomains(host)
     not_domains = get_not_subdomains(host)
 
     ports_ = {}
     for scheme, ports_used in ports.iteritems():
         ports_[scheme] = ports_used
 
     for key, value in domains.iteritems():
@@ -670,16 +675,18 @@ def normalise_config(config, ports):
 
     # make a (shallow) copy of the config and update that, so that the
     # normalized config can be used in place of the original one.
     config_ = config.copy()
     config_["domains"] = domains
     config_["not_domains"] = not_domains
     config_["ports"] = ports_
     config_["bind_address"] = bind_address
+    if config.get("server_host", None) is None:
+        config_["server_host"] = host
     return config_
 
 
 def get_paths(config):
     return {"doc_root": config["doc_root"],
             "ws_doc_root": config["ws_doc_root"]}
 
 
@@ -687,17 +694,17 @@ def get_ssl_config(config, ssl_environme
     external_domains = config["domains"].values()
     key_path, cert_path = ssl_environment.host_cert_path(external_domains)
     return {"key_path": key_path,
             "cert_path": cert_path,
             "encrypt_after_connect": config["ssl"]["encrypt_after_connect"]}
 
 
 def start(config, ssl_environment, routes, **kwargs):
-    host = config.get("host_ip") or config["host"]
+    host = config["server_host"]
     ports = get_ports(config, ssl_environment)
     paths = get_paths(config)
     bind_address = config["bind_address"]
     ssl_config = get_ssl_config(config, ssl_environment)
 
     servers = start_servers(host, ports, paths, routes, bind_address, config,
                             ssl_config, **kwargs)
 
@@ -810,17 +817,17 @@ def run(**kwargs):
                          os.path.join(repo_root, "config.json"),
                          **kwargs)
 
     setup_logger(config["log_level"])
 
     with get_ssl_environment(config) as ssl_env:
         ports = get_ports(config, ssl_env)
         config = normalise_config(config, ports)
-        host = config["host"]
+        host = config["browser_host"]
         bind_address = config["bind_address"]
 
         if config["check_subdomains"]:
             paths = get_paths(config)
             ssl_config = get_ssl_config(config, ssl_env)
             check_subdomains(host, paths, bind_address, ssl_config, config["aliases"])
 
         stash_address = None
--- a/testing/web-platform/tests/tools/wpt/run.py
+++ b/testing/web-platform/tests/tools/wpt/run.py
@@ -91,20 +91,21 @@ your PATH.
 Otherwise run with --ssl-type=none""")
             else:
                 raise WptrunError("""OpenSSL not found. If you don't need HTTPS support run with --ssl-type=none,
 otherwise install OpenSSL and ensure that it's on your $PATH.""")
 
 
 def check_environ(product):
     if product not in ("firefox", "servo"):
-        expected_hosts = {".".join(x)
-                          for x in serve.get_subdomains("web-platform.test").values()}
-        expected_hosts |= {".".join(x)
-                           for x in serve.get_not_subdomains("web-platform.test").values()}
+        config = serve.load_config(os.path.join(wpt_root, "config.default.json"),
+                                   os.path.join(wpt_root, "config.json"))
+        config = serve.normalise_config(config, {})
+        expected_hosts = (set(config["domains"].itervalues()) ^
+                          set(config["not_domains"].itervalues()))
         missing_hosts = set(expected_hosts)
         if platform.uname()[0] != "Windows":
             hosts_path = "/etc/hosts"
         else:
             hosts_path = "C:\Windows\System32\drivers\etc\hosts"
         with open(hosts_path, "r") as f:
             for line in f:
                 line = line.split("#", 1)[0].strip()
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/firefox.py
@@ -117,20 +117,18 @@ def env_extras(**kwargs):
 
 
 def env_options():
     # The server host is set to 127.0.0.1 as Firefox is configured (through the
     # network.dns.localDomains preference set below) to resolve the test
     # domains to localhost without relying on the network stack.
     #
     # https://github.com/w3c/web-platform-tests/pull/9480
-    return {"host_ip": "127.0.0.1",
-            "host": "web-platform.test",
+    return {"server_host": "127.0.0.1",
             "bind_address": False,
-            "certificate_domain": "web-platform.test",
             "supports_debugger": True}
 
 
 def run_info_extras(**kwargs):
     return {"e10s": kwargs["gecko_e10s"],
             "headless": "MOZ_HEADLESS" in os.environ}
 
 
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servo.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servo.py
@@ -47,18 +47,17 @@ def executor_kwargs(test_type, server_co
     return rv
 
 
 def env_extras(**kwargs):
     return []
 
 
 def env_options():
-    return {"host": "web-platform.test",
-            "host_ip": "127.0.0.1",
+    return {"server_host": "127.0.0.1",
             "bind_address": False,
             "testharnessreport": "testharnessreport-servo.js",
             "supports_debugger": True}
 
 
 def update_properties():
     return ["debug", "os", "version", "processor", "bits"], None
 
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servodriver.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/browsers/servodriver.py
@@ -48,18 +48,17 @@ def executor_kwargs(test_type, server_co
     return rv
 
 
 def env_extras(**kwargs):
     return []
 
 
 def env_options():
-    return {"host_ip": "127.0.0.1",
-            "host": "web-platform.test",
+    return {"server_host": "127.0.0.1",
             "testharnessreport": "testharnessreport-servodriver.js",
             "supports_debugger": True}
 
 
 def update_properties():
     return ["debug", "os", "version", "processor", "bits"], None
 
 
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/environment.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/environment.py
@@ -142,35 +142,35 @@ class TestEnvironment(object):
                 "http": [8000, 8001],
                 "https": [8443],
                 "ws": [8888]
             },
             "check_subdomains": False,
             "ssl": {}
         }
 
-        if "host" in self.options:
-            local_config["host"] = self.options["host"]
+        if "browser_host" in self.options:
+            local_config["browser_host"] = self.options["browser_host"]
 
         if "bind_address" in self.options:
             local_config["bind_address"] = self.options["bind_address"]
 
         with open(default_config_path) as f:
             default_config = json.load(f)
 
-        local_config["host_ip"] = self.options.get("host_ip", None)
+        local_config["server_host"] = self.options.get("server_host", None)
         local_config["ssl"]["encrypt_after_connect"] = self.options.get("encrypt_after_connect", False)
 
         config = serve.merge_json(default_config, local_config)
         config["doc_root"] = serve_path(self.test_paths)
 
         if not self.ssl_env.ssl_enabled:
             config["ports"]["https"] = [None]
 
-        host = self.options.get("certificate_domain", config["host"])
+        host = config["browser_host"]
         hosts = [host]
         hosts.extend("%s.%s" % (item[0], host) for item in serve.get_subdomains(host).values())
         key_file, certificate = self.ssl_env.host_cert_path(hosts)
 
         config["key_file"] = key_file
         config["certificate"] = certificate
 
         serve.set_computed_defaults(config)
@@ -235,17 +235,17 @@ class TestEnvironment(object):
             if not failed:
                 return
             time.sleep(0.5)
         raise EnvironmentError("Servers failed to start: %s" %
                                ", ".join("%s:%s" % item for item in failed))
 
     def test_servers(self):
         failed = []
-        host = self.config.get("host_ip") or self.config.get("host")
+        host = self.config["server_host"]
         for scheme, servers in self.servers.iteritems():
             for port, server in servers:
                 if self.test_server_port:
                     s = socket.socket()
                     try:
                         s.connect((host, port))
                     except socket.error:
                         failed.append((host, port))
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/base.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/base.py
@@ -167,17 +167,17 @@ class TestExecutor(object):
             self.logger.debug(result[0].message)
 
         self.last_environment = test.environment
 
         self.runner.send_message("test_ended", test, result)
 
     def server_url(self, protocol):
         return "%s://%s:%s" % (protocol,
-                               self.server_config["host"],
+                               self.server_config["browser_host"],
                                self.server_config["ports"][protocol][0])
 
     def test_url(self, test):
         return urlparse.urljoin(self.server_url(test.environment["protocol"]), test.url)
 
     @abstractmethod
     def do_test(self, test):
         """Test-type and protocol specific implementation of running a
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_products.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/tests/test_products.py
@@ -53,11 +53,13 @@ def test_server_start_config(product):
                                          sslutils.environments["none"](None),
                                          False,
                                          None,
                                          env_options,
                                          env_extras) as test_environment:
             start.assert_called_once()
             args = start.call_args
             config = args[0][0]
-            if "host" in env_options:
-                assert config["host"] == env_options["host"]
+            if "server_host" in env_options:
+                assert config["server_host"] == env_options["server_host"]
+            else:
+                assert config["server_host"] == config["browser_host"]
             assert isinstance(config["bind_address"], bool)
--- a/testing/web-platform/tests/tools/wptserve/wptserve/pipes.py
+++ b/testing/web-platform/tests/tools/wptserve/wptserve/pipes.py
@@ -393,16 +393,18 @@ def template(request, content, escape_ty
         elif field == "GET":
             value = FirstWrapper(request.GET)
         elif field == "domains":
             if ('not_domains' in request.server.config and
                     tokens[1][1] in request.server.config['not_domains']):
                 value = request.server.config['not_domains']
             else:
                 value = request.server.config['domains']
+        elif field == "host":
+            value = request.server.config["browser_host"]
         elif field in request.server.config:
             value = request.server.config[tokens[0][1]]
         elif field == "location":
             value = {"server": "%s://%s:%s" % (request.url_parts.scheme,
                                                request.url_parts.hostname,
                                                request.url_parts.port),
                      "scheme": request.url_parts.scheme,
                      "host": "%s:%s" % (request.url_parts.hostname,
--- a/testing/web-platform/tests/tools/wptserve/wptserve/server.py
+++ b/testing/web-platform/tests/tools/wptserve/wptserve/server.py
@@ -163,17 +163,18 @@ class WebTestServer(ThreadingMixIn, Base
 
         #super doesn't work here because BaseHTTPServer.HTTPServer is old-style
         BaseHTTPServer.HTTPServer.__init__(self, hostname_port, request_handler_cls, **kwargs)
 
         if config is not None:
             Server.config = config
         else:
             self.logger.debug("Using default configuration")
-            Server.config = {"host": server_address[0],
+            Server.config = {"browser_host": server_address[0],
+                             "server_host": server_address[0],
                              "domains": {"": server_address[0]},
                              "ports": {"http": [self.server_address[1]]}}
 
 
         self.key_file = key_file
         self.certificate = certificate
         self.encrypt_after_connect = use_ssl and encrypt_after_connect