Bug 1150821 - Update to latest wptrunner, a=testonly
authorJames Graham <james@hoppipolla.co.uk>
Thu, 02 Apr 2015 14:46:51 +0100
changeset 237525 4d7f9a62a3658b74acdef53fb815f4a12229d29c
parent 237524 890e0a621055c31b1cdfec1015fcfc0865a2b4e0
child 237526 f2f5bb353be9f5f1b42255ed52d35a492e4ba1ff
push id57964
push userjames@hoppipolla.co.uk
push dateFri, 03 Apr 2015 09:17:04 +0000
treeherdermozilla-inbound@ca07ab6717ce [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1150821
milestone40.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1150821 - Update to latest wptrunner, a=testonly
testing/web-platform/harness/wptrunner/browsers/b2g_setup/certtest_app.zip
testing/web-platform/harness/wptrunner/executors/executorservo.py
testing/web-platform/harness/wptrunner/update/metadata.py
testing/web-platform/harness/wptrunner/update/sync.py
testing/web-platform/harness/wptrunner/update/update.py
testing/web-platform/harness/wptrunner/wptcommandline.py
index ade8879b212871878e63586598b7bca555b4c2ba..f9cbd5300ad650f8369ab02a4adc8866a5f0dea1
GIT binary patch
literal 1237
zc$^FHW@Zs#U|`^2FjvWODfzT}tqYJB1;ktoG7On{DXA5D86~+np&^_M%$mjW!KOD9
zgG(#885mi<GBPlL^==OIIq9#rQpY3ogvQCU8y|0K3|!f%S!Sj>Q`K^1nb|J`DV<ZP
zlc&rN$~?Yw!Gk$bAwrX)VirA!2sn_sm|u`>6_Ye0Ljc?zViomnVjn)nc``FF?BZfz
z;0L-RH!&|WEw#8ruRJv=v7i9#7#TOIP<bFmbIjZ@-~7iGJbS-~b6EDzs?K-SufA8-
z$ST&{sC03ofTxIx)jXez|MyLKd!*=b*`=K<1}D|)Y;B)@_3Q8rkTc^?`pnW9r!cvq
zr||RIiWeg6{b~KVIx{ZjD7;Qlj^yEvnfzmk+-1iMo+UXACQNf)HRaX2xaKnE<QjbV
zyifK0Id-{&iBCSTe>~N;YQ=Q1Ba2l6>NryUr^gg-aWS;d*}m8IMs@%GHN~&wF4z6O
zA?W2)QhB&xyCS3f+uU<81@nS6mQRx3cYoPpQNfmtYj0kY><(J(5!>YY;YCI$$Ac#Y
zH%xOrO{@HHQvMgS+WI@{?~Ll!Uh|&aG>P}O(Lw$@EPP4b4?|qvOtE_-Tz2pTk7Dwv
z)#rF#W|&^FyUy6ZV@>fk)i<9PF`TvbS!LExEVHa9Y^U5Y2XS9j>*;GvpKO}u%ar;n
zh12Yj%*iX(0>}E*k`&Yuv?XqSbI)cFKX<A1OpE8+hl<6bC)X^w@@Mmzod;Z3bBpF_
zl`+b^`e}8h{N^kD#UVY}D-CvB?#ZkPIx{iLt=geIJbCM}tGBl+mVNmbT_tK!m}++*
zv;A?$d9{A)*A3TY9rm9#+{F|iIN_9Cq>8u8f||D{mT1JQ^PjzSX4>20Gc0Q&Pg{wk
zF!i%+N?&?tUy)u*%C=8?n=bs6U2-~hW~)-teh-1&G52TlxYU&9E4Y0ty(Hr(tH66}
z(gv~qPUfvor*Qt+U;j7uric5?K=}o6PiJ{e_}X39cY?Pn_4O3y?bky#r7Q9M4O6=z
zS{C<y&2#P8Nc~r_5-%47<;7jFKR^BaR`oZT9~tH@ef%su`@k7iiHwlvFOEleIQEP3
z_?_o|+~r;QzjtTc^6gLmMwiE>aPH7paYRB@RkX-h=dwFH%Xa7884+t!e0KN?m*z2V
z3TU!5`kZoDeEo$L8~<?4eg66Ap`TJ}zj)HOK3l+SenQ%8zH8RwsaqBv40;rzR@1dY
z&uD+~uBpe;dfh$ZTP)9L+ZO!!o_Kk4UTDmhgf&{RPiIYdfAZKrJ)ieeU+8X4I?!1p
z=+f-T^2W8~)sf!<OTW+DzH;`C|7M#$t+{*U?mW>WeOHAPIAX&mZLjHiP}=d_QK4@3
zmrZ}BXTGn$@Z)LE&dPtE6TKd{cKkGdJKekTd)asU0B=SnIc8jCg9HNz05cuKl12~<
sxujr)loV(g6WKh>JdSK$2av?b@vM*>k6|b)8%QG)5H1DMQY;`I0L69!3IG5A
--- a/testing/web-platform/harness/wptrunner/executors/executorservo.py
+++ b/testing/web-platform/harness/wptrunner/executors/executorservo.py
@@ -1,12 +1,13 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import base64
 import hashlib
 import json
 import os
 import subprocess
 import tempfile
 import threading
 import urlparse
 import uuid
@@ -16,74 +17,103 @@ from mozprocess import ProcessHandler
 
 from .base import (ExecutorException,
                    Protocol,
                    RefTestImplementation,
                    testharness_result_converter,
                    reftest_result_converter)
 from .process import ProcessTestExecutor
 
+hosts_text = """127.0.0.1 web-platform.test
+127.0.0.1 www.web-platform.test
+127.0.0.1 www1.web-platform.test
+127.0.0.1 www2.web-platform.test
+127.0.0.1 xn--n8j6ds53lwwkrqhv28a.web-platform.test
+127.0.0.1 xn--lve-6lad.web-platform.test
+"""
+
+def make_hosts_file():
+    hosts_fd, hosts_path = tempfile.mkstemp()
+    with os.fdopen(hosts_fd, "w") as f:
+        f.write(hosts_text)
+    return hosts_path
 
 class ServoTestharnessExecutor(ProcessTestExecutor):
     convert_result = testharness_result_converter
 
     def __init__(self, browser, server_config, timeout_multiplier=1, debug_args=None,
                  pause_after_test=False):
         ProcessTestExecutor.__init__(self, browser, server_config,
                                      timeout_multiplier=timeout_multiplier,
                                      debug_args=debug_args)
         self.pause_after_test = pause_after_test
         self.result_data = None
         self.result_flag = None
         self.protocol = Protocol(self, browser)
+        self.hosts_path = make_hosts_file()
+
+    def teardown(self):
+        try:
+            os.unlink(self.hosts_path)
+        except OSError:
+            pass
+        ProcessTestExecutor.teardown(self)
 
     def do_test(self, test):
         self.result_data = None
         self.result_flag = threading.Event()
 
         self.command = [self.binary, "--cpu", "--hard-fail", "-z", self.test_url(test)]
 
         if self.pause_after_test:
             self.command.remove("-z")
 
         if self.debug_args:
             self.command = list(self.debug_args) + self.command
 
+        env = os.environ.copy()
+        env["HOST_FILE"] = self.hosts_path
 
         self.proc = ProcessHandler(self.command,
                                    processOutputLine=[self.on_output],
-                                   onFinish=self.on_finish)
-        self.proc.run()
+                                   onFinish=self.on_finish,
+                                   env=env)
 
-        timeout = test.timeout * self.timeout_multiplier
+        try:
+            self.proc.run()
 
-        # Now wait to get the output we expect, or until we reach the timeout
-        if self.debug_args is None and not self.pause_after_test:
-            wait_timeout = timeout + 5
-        else:
-            wait_timeout = None
-        self.result_flag.wait(wait_timeout)
+            timeout = test.timeout * self.timeout_multiplier
+
+            # Now wait to get the output we expect, or until we reach the timeout
+            if self.debug_args is None and not self.pause_after_test:
+                wait_timeout = timeout + 5
+            else:
+                wait_timeout = None
+            self.result_flag.wait(wait_timeout)
 
-        proc_is_running = True
-        if self.result_flag.is_set() and self.result_data is not None:
-            self.result_data["test"] = test.url
-            result = self.convert_result(test, self.result_data)
-        else:
-            if self.proc.proc.poll() is not None:
-                result = (test.result_cls("CRASH", None), [])
-                proc_is_running = False
+            proc_is_running = True
+            if self.result_flag.is_set() and self.result_data is not None:
+                self.result_data["test"] = test.url
+                result = self.convert_result(test, self.result_data)
             else:
-                result = (test.result_cls("TIMEOUT", None), [])
+                if self.proc.proc.poll() is not None:
+                    result = (test.result_cls("CRASH", None), [])
+                    proc_is_running = False
+                else:
+                    result = (test.result_cls("TIMEOUT", None), [])
 
-        if proc_is_running:
-            if self.pause_after_test:
-                self.logger.info("Pausing until the browser exits")
-                self.proc.wait()
-            else:
-                self.proc.kill()
+            if proc_is_running:
+                if self.pause_after_test:
+                    self.logger.info("Pausing until the browser exits")
+                    self.proc.wait()
+                else:
+                    self.proc.kill()
+        except KeyboardInterrupt:
+            self.proc.kill()
+            raise
 
         return result
 
     def on_output(self, line):
         prefix = "ALERT: RESULT: "
         line = line.decode("utf8", "replace")
         if line.startswith(prefix):
             self.result_data = json.loads(line[len(prefix):])
@@ -116,53 +146,69 @@ class TempFilename(object):
             pass
 
 
 class ServoRefTestExecutor(ProcessTestExecutor):
     convert_result = reftest_result_converter
 
     def __init__(self, browser, server_config, binary=None, timeout_multiplier=1,
                  screenshot_cache=None, debug_args=None, pause_after_test=False):
+
         ProcessTestExecutor.__init__(self,
                                      browser,
                                      server_config,
                                      timeout_multiplier=timeout_multiplier,
                                      debug_args=debug_args)
 
         self.protocol = Protocol(self, browser)
         self.screenshot_cache = screenshot_cache
         self.implementation = RefTestImplementation(self)
         self.tempdir = tempfile.mkdtemp()
+        self.hosts_path = make_hosts_file()
 
     def teardown(self):
+        try:
+            os.unlink(self.hosts_path)
+        except OSError:
+            pass
         os.rmdir(self.tempdir)
         ProcessTestExecutor.teardown(self)
 
     def screenshot(self, test):
         full_url = self.test_url(test)
 
         with TempFilename(self.tempdir) as output_path:
             self.command = [self.binary, "--cpu", "--hard-fail", "--exit",
                             "--output=%s" % output_path, full_url]
 
+            env = os.environ.copy()
+            env["HOST_FILE"] = self.hosts_path
+
             self.proc = ProcessHandler(self.command,
-                                       processOutputLine=[self.on_output])
-            self.proc.run()
-            rv = self.proc.wait(timeout=test.timeout)
+                                       processOutputLine=[self.on_output],
+                                       env=env)
+
+            try:
+                self.proc.run()
+                rv = self.proc.wait(timeout=test.timeout)
+            except KeyboardInterrupt:
+                self.proc.kill()
+                raise
+
             if rv is None:
                 self.proc.kill()
                 return False, ("EXTERNAL-TIMEOUT", None)
 
             if rv != 0 or not os.path.exists(output_path):
                 return False, ("CRASH", None)
 
             with open(output_path) as f:
                 # Might need to strip variable headers or something here
                 data = f.read()
-                return True, data
+                return True, base64.b64encode(data)
 
     def do_test(self, test):
         result = self.implementation.run_test(test)
 
         return self.convert_result(test, result)
 
     def on_output(self, line):
         line = line.decode("utf8", "replace")
--- a/testing/web-platform/harness/wptrunner/update/metadata.py
+++ b/testing/web-platform/harness/wptrunner/update/metadata.py
@@ -34,20 +34,20 @@ class CreateMetadataPatch(Step):
         if state.no_patch:
             return
 
         local_tree = state.local_tree
         sync_tree = state.sync_tree
 
         if sync_tree is not None:
             name = "web-platform-tests_update_%s_metadata" % sync_tree.rev
-            message = "Update web-platform-tests expected data to revision %s" % sync_tree.rev
+            message = "Update %s expected data to revision %s" % (state.suite_name, sync_tree.rev)
         else:
             name = "web-platform-tests_update_metadata"
-            message = "Update web-platform-tests expected data"
+            message = "Update %s expected data" % state.suite_name
 
         local_tree.create_patch(name, message)
 
         if not local_tree.is_clean:
             metadata_paths = [manifest_path["metadata_path"]
                               for manifest_path in state.paths.itervalues()]
             for path in metadata_paths:
                 local_tree.add_new(os.path.relpath(path, local_tree.root))
--- a/testing/web-platform/harness/wptrunner/update/sync.py
+++ b/testing/web-platform/harness/wptrunner/update/sync.py
@@ -149,17 +149,17 @@ class CreateSyncPatch(Step):
     def create(self, state):
         if state.no_patch:
             return
 
         local_tree = state.local_tree
         sync_tree = state.sync_tree
 
         local_tree.create_patch("web-platform-tests_update_%s" % sync_tree.rev,
-                                "Update web-platform-tests to revision %s" % sync_tree.rev)
+                                "Update %s to revision %s" % (state.suite_name, sync_tree.rev))
         local_tree.add_new(os.path.relpath(state.tests_path,
                                            local_tree.root))
         updated = local_tree.update_patch(include=[state.tests_path,
                                                    state.metadata_path])
         local_tree.commit_patch()
 
         if not updated:
             self.logger.info("Nothing to sync")
--- a/testing/web-platform/harness/wptrunner/update/update.py
+++ b/testing/web-platform/harness/wptrunner/update/update.py
@@ -8,18 +8,19 @@ import sys
 from metadata import MetadataUpdateRunner
 from sync import SyncFromUpstreamRunner
 from tree import GitTree, HgTree, NoVCSTree
 
 from .. import environment as env
 from base import Step, StepRunner, exit_clean, exit_unclean
 from state import State
 
-def setup_paths(logger, test_paths):
-    env.do_delayed_imports(logger, test_paths)
+def setup_paths(sync_path):
+    sys.path.insert(0, sync_path)
+    from tools import localpaths
 
 class LoadConfig(Step):
     """Step for loading configuration from the ini file and kwargs."""
 
     provides = ["sync", "paths", "metadata_path", "tests_path"]
 
     def create(self, state):
         state.sync = {"remote_url": state.kwargs["remote_url"],
@@ -63,35 +64,38 @@ class SyncFromUpstream(Step):
         if not state.kwargs["sync"]:
             return
 
         if not state.sync_tree:
             os.mkdir(state.sync["path"])
             state.sync_tree = GitTree(root=state.sync["path"])
 
         kwargs = state.kwargs
-        with state.push(["sync", "paths", "metadata_path", "tests_path", "local_tree", "sync_tree"]):
+        with state.push(["sync", "paths", "metadata_path", "tests_path", "local_tree",
+                         "sync_tree"]):
             state.target_rev = kwargs["rev"]
             state.no_patch = kwargs["no_patch"]
+            state.suite_name = kwargs["suite_name"]
             runner = SyncFromUpstreamRunner(self.logger, state)
             runner.run()
 
 
 class UpdateMetadata(Step):
     """Update the expectation metadata from a set of run logs"""
 
     def create(self, state):
         if not state.kwargs["run_log"]:
             return
 
         kwargs = state.kwargs
         with state.push(["local_tree", "sync_tree", "paths", "serve_root"]):
             state.run_log = kwargs["run_log"]
             state.ignore_existing = kwargs["ignore_existing"]
             state.no_patch = kwargs["no_patch"]
+            state.suite_name = kwargs["suite_name"]
             runner = MetadataUpdateRunner(self.logger, state)
             runner.run()
 
 
 class UpdateRunner(StepRunner):
     """Runner for doing an overall update."""
     steps = [LoadConfig,
              LoadTrees,
@@ -104,18 +108,21 @@ class WPTUpdate(object):
         """Object that controls the running of a whole wptupdate.
 
         :param runner_cls: Runner subclass holding the overall list of
                            steps to run.
         :param kwargs: Command line arguments
         """
         self.runner_cls = runner_cls
         self.serve_root = kwargs["test_paths"]["/"]["tests_path"]
-        #This must be before we try to reload state
-        setup_paths(logger, kwargs["test_paths"])
+
+        if not kwargs["sync"]:
+            setup_paths(self.serve_root)
+        else:
+            setup_paths(kwargs["sync_path"])
 
         self.state = State(logger)
         self.kwargs = kwargs
         self.logger = logger
 
     def run(self, **kwargs):
         if self.kwargs["abort"]:
             self.abort()
--- a/testing/web-platform/harness/wptrunner/wptcommandline.py
+++ b/testing/web-platform/harness/wptrunner/wptcommandline.py
@@ -61,34 +61,34 @@ def create_parser(product_choices=None):
 
     parser.add_argument("--manifest-update", action="store_true", default=False,
                         help="Force regeneration of the test manifest")
 
     parser.add_argument("--binary", action="store",
                         type=abs_path, help="Binary to run tests against")
     parser.add_argument("--webdriver-binary", action="store", metavar="BINARY",
                         type=abs_path, help="WebDriver server binary to use")
-    parser.add_argument("--processes", action="store", type=int, default=1,
+    parser.add_argument("--processes", action="store", type=int, default=None,
                         help="Number of simultaneous processes to use")
 
     parser.add_argument("--run-by-dir", type=int, nargs="?", default=False,
                         help="Split run into groups by directories. With a parameter,"
                         "limit the depth of splits e.g. --run-by-dir=1 to split by top-level"
                         "directory")
 
     parser.add_argument("--timeout-multiplier", action="store", type=float, default=None,
                         help="Multiplier relative to standard test timeout to use")
     parser.add_argument("--repeat", action="store", type=int, default=1,
                         help="Number of times to run the tests")
 
     parser.add_argument("--no-capture-stdio", action="store_true", default=False,
                         help="Don't capture stdio and write to logging")
 
     parser.add_argument("--product", action="store", choices=product_choices,
-                        default="firefox", help="Browser against which to run tests")
+                        default=None, help="Browser against which to run tests")
 
     parser.add_argument("--list-test-groups", action="store_true",
                         default=False,
                         help="List the top level directories containing tests that will run.")
     parser.add_argument("--list-disabled", action="store_true",
                         default=False,
                         help="List the tests that are disabled on the current platform")
 
@@ -165,16 +165,17 @@ def create_parser(product_choices=None):
 
 def set_from_config(kwargs):
     if kwargs["config"] is None:
         config_path = config.path()
     else:
         config_path = kwargs["config"]
 
     kwargs["config_path"] = config_path
+
     kwargs["config"] = config.read(kwargs["config_path"])
 
     keys = {"paths": [("prefs", "prefs_root", True),
                       ("run_info", "run_info", True)],
             "web-platform-tests": [("remote_url", "remote_url", False),
                                    ("branch", "branch", False),
                                    ("sync_path", "sync_path", True)],
             "SSL": [("openssl_binary", "openssl_binary", True),
@@ -199,16 +200,18 @@ def set_from_config(kwargs):
             kwargs["test_paths"]["/"] = {}
         kwargs["test_paths"]["/"]["tests_path"] = kwargs["tests_root"]
 
     if kwargs["metadata_root"]:
         if "/" not in kwargs["test_paths"]:
             kwargs["test_paths"]["/"] = {}
         kwargs["test_paths"]["/"]["metadata_path"] = kwargs["metadata_root"]
 
+    kwargs["suite_name"] = kwargs["config"].get("web-platform-tests", {}).get("name", "web-platform-tests")
+
 
 def get_test_paths(config):
     # Set up test_paths
     test_paths = OrderedDict()
 
     for section in config.iterkeys():
         if section.startswith("manifest:"):
             manifest_opts = config.get(section)
@@ -245,16 +248,19 @@ def check_args(kwargs):
             if not os.path.exists(path):
                 print "Fatal: %s path %s does not exist" % (name, path)
                 sys.exit(1)
 
             if not os.path.isdir(path):
                 print "Fatal: %s path %s is not a directory" % (name, path)
                 sys.exit(1)
 
+    if kwargs["product"] is None:
+        kwargs["product"] = "firefox"
+
     if kwargs["test_list"]:
         if kwargs["include"] is not None:
             kwargs["include"].extend(kwargs["test_list"])
         else:
             kwargs["include"] = kwargs["test_list"]
 
     if kwargs["run_info"] is None:
         kwargs["run_info"] = kwargs["config_path"]
@@ -263,16 +269,19 @@ def check_args(kwargs):
         require_arg(kwargs, "total_chunks", lambda x: x >= kwargs["this_chunk"])
 
     if kwargs["chunk_type"] is None:
         if kwargs["total_chunks"] > 1:
             kwargs["chunk_type"] = "equal_time"
         else:
             kwargs["chunk_type"] = "none"
 
+    if kwargs["processes"] is None:
+        kwargs["processes"] = 1
+
     if kwargs["debugger"] is not None:
         debug_args, interactive = debugger_arguments(kwargs["debugger"],
                                                      kwargs["debugger_args"])
         if interactive:
             require_arg(kwargs, "processes", lambda x: x == 1)
             kwargs["no_capture_stdio"] = True
         kwargs["interactive"] = interactive
         kwargs["debug_args"] = debug_args