Bug 1257799 - Update to latest wptrunner, a=testonly
authorJames Graham <james@hoppipolla.co.uk>
Wed, 16 Mar 2016 13:55:15 +0000
changeset 289289 8938d99b75bb12ce23edc53f90c87b2d7ba1071d
parent 289288 6ab07b5c4c3eab8e76793e6ff630a94b59442991
child 289290 39ab55ac2fc8017c986dcf8629129379f2e9351b
push id73789
push userjames@hoppipolla.co.uk
push dateFri, 18 Mar 2016 11:39:56 +0000
treeherdermozilla-inbound@95f97693b04f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1257799
milestone48.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1257799 - Update to latest wptrunner, a=testonly MozReview-Commit-ID: K7Ro7yG0Nds
testing/web-platform/harness/docs/expectation.rst
testing/web-platform/harness/wptrunner/browsers/firefox.py
testing/web-platform/harness/wptrunner/executors/base.py
testing/web-platform/harness/wptrunner/executors/executormarionette.py
testing/web-platform/harness/wptrunner/executors/executorselenium.py
testing/web-platform/harness/wptrunner/executors/executorservo.py
testing/web-platform/harness/wptrunner/executors/executorservodriver.py
testing/web-platform/harness/wptrunner/manifestexpected.py
testing/web-platform/harness/wptrunner/testrunner.py
testing/web-platform/harness/wptrunner/wptcommandline.py
testing/web-platform/harness/wptrunner/wptrunner.py
testing/web-platform/harness/wptrunner/wpttest.py
--- a/testing/web-platform/harness/docs/expectation.rst
+++ b/testing/web-platform/harness/docs/expectation.rst
@@ -198,20 +198,16 @@ When used for expectation data, manifest
    reference.
 
  * A key ``expected`` giving the expectation value of each (sub)test.
 
  * A key ``disabled`` which can be set to any value to indicate that
    the (sub)test is disabled and should either not be run (for tests)
    or that its results should be ignored (subtests).
 
- * A key ``restart-after`` which can be set to any value to indicate that
-   the runner should restart the browser after running this test (e.g. to
-   clear out unwanted state).
-
  * Variables ``debug``, ``os``, ``version``, ``processor`` and
    ``bits`` that describe the configuration of the browser under
    test. ``debug`` is a boolean indicating whether a build is a debug
    build. ``os`` is a string indicating the operating system, and
    ``version`` a string indicating the particular version of that
    operating system. ``processor`` is a string indicating the
    processor architecture and ``bits`` an integer indicating the
    number of bits. This information is typically provided by
--- a/testing/web-platform/harness/wptrunner/browsers/firefox.py
+++ b/testing/web-platform/harness/wptrunner/browsers/firefox.py
@@ -77,16 +77,17 @@ def run_info_extras(**kwargs):
     return {"e10s": kwargs["gecko_e10s"]}
 
 
 def update_properties():
     return ["debug", "e10s", "os", "version", "processor", "bits"], {"debug", "e10s"}
 
 class FirefoxBrowser(Browser):
     used_ports = set()
+    init_timeout = 60
 
     def __init__(self, logger, binary, prefs_root, debug_info=None,
                  symbols_path=None, stackwalk_binary=None, certutil_binary=None,
                  ca_certificate_path=None, e10s=False):
         Browser.__init__(self, logger)
         self.binary = binary
         self.prefs_root = prefs_root
         self.marionette_port = None
--- a/testing/web-platform/harness/wptrunner/executors/base.py
+++ b/testing/web-platform/harness/wptrunner/executors/base.py
@@ -201,92 +201,96 @@ class RefTestImplementation(object):
         # retrieve the screenshot from the cache directly in the future
         self.screenshot_cache = self.executor.screenshot_cache
         self.message = None
 
     @property
     def logger(self):
         return self.executor.logger
 
-    def get_hash(self, test):
+    def get_hash(self, test, viewport_size, dpi):
         timeout = test.timeout * self.timeout_multiplier
+        key = (test.url, viewport_size, dpi)
 
-        if test.url not in self.screenshot_cache:
-            success, data = self.executor.screenshot(test)
+        if key not in self.screenshot_cache:
+            success, data = self.executor.screenshot(test, viewport_size, dpi)
 
             if not success:
                 return False, data
 
             screenshot = data
             hash_value = hashlib.sha1(screenshot).hexdigest()
 
-            self.screenshot_cache[test.url] = (hash_value, None)
+            self.screenshot_cache[key] = (hash_value, None)
 
-            rv = True, (hash_value, screenshot)
+            rv = (hash_value, screenshot)
         else:
-            rv = True, self.screenshot_cache[test.url]
+            rv = self.screenshot_cache[key]
 
-        self.message.append("%s %s" % (test.url, rv[1][0]))
-        return rv
+        self.message.append("%s %s" % (test.url, rv[0]))
+        return True, rv
 
     def is_pass(self, lhs_hash, rhs_hash, relation):
         assert relation in ("==", "!=")
         self.message.append("Testing %s %s %s" % (lhs_hash, relation, rhs_hash))
         return ((relation == "==" and lhs_hash == rhs_hash) or
                 (relation == "!=" and lhs_hash != rhs_hash))
 
     def run_test(self, test):
+        viewport_size = test.viewport_size
+        dpi = test.dpi
         self.message = []
 
         # Depth-first search of reference tree, with the goal
         # of reachings a leaf node with only pass results
 
         stack = list(((test, item[0]), item[1]) for item in reversed(test.references))
         while stack:
             hashes = [None, None]
             screenshots = [None, None]
 
             nodes, relation = stack.pop()
 
             for i, node in enumerate(nodes):
-                success, data = self.get_hash(node)
+                success, data = self.get_hash(node, viewport_size, dpi)
                 if success is False:
                     return {"status": data[0], "message": data[1]}
 
                 hashes[i], screenshots[i] = data
 
             if self.is_pass(hashes[0], hashes[1], relation):
                 if nodes[1].references:
                     stack.extend(list(((nodes[1], item[0]), item[1]) for item in reversed(nodes[1].references)))
                 else:
                     # We passed
                     return {"status":"PASS", "message": None}
 
         # We failed, so construct a failure message
 
         for i, (node, screenshot) in enumerate(zip(nodes, screenshots)):
             if screenshot is None:
-                success, screenshot = self.retake_screenshot(node)
+                success, screenshot = self.retake_screenshot(node, viewport_size, dpi)
                 if success:
                     screenshots[i] = screenshot
 
         log_data = [{"url": nodes[0].url, "screenshot": screenshots[0]}, relation,
                     {"url": nodes[1].url, "screenshot": screenshots[1]}]
 
         return {"status": "FAIL",
                 "message": "\n".join(self.message),
                 "extra": {"reftest_screenshots": log_data}}
 
-    def retake_screenshot(self, node):
-        success, data = self.executor.screenshot(node)
+    def retake_screenshot(self, node, viewport_size, dpi):
+        success, data = self.executor.screenshot(node, viewport_size, dpi)
         if not success:
             return False, data
 
-        hash_val, _ = self.screenshot_cache[node.url]
-        self.screenshot_cache[node.url] = hash_val, data
+        key = (node.url, viewport_size, dpi)
+        hash_val, _ = self.screenshot_cache[key]
+        self.screenshot_cache[key] = hash_val, data
         return True, data
 
 class Protocol(object):
     def __init__(self, executor, browser):
         self.executor = executor
         self.browser = browser
 
     @property
--- a/testing/web-platform/harness/wptrunner/executors/executormarionette.py
+++ b/testing/web-platform/harness/wptrunner/executors/executormarionette.py
@@ -102,22 +102,16 @@ class MarionetteProtocol(Protocol):
         try:
             # Get a simple property over the connection
             self.marionette.current_window_handle
         except Exception:
             return False
         return True
 
     def after_connect(self):
-        # Turn off debug-level logging by default since this is so verbose
-        with self.marionette.using_context("chrome"):
-            self.marionette.execute_script("""
-              Components.utils.import("resource://gre/modules/Log.jsm");
-              Log.repository.getLogger("Marionette").level = Log.Level.Info;
-            """)
         self.load_runner("http")
 
     def load_runner(self, protocol):
         # Check if we previously had a test window open, and if we did make sure it's closed
         self.marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
         url = urlparse.urljoin(self.executor.server_url(protocol), "/testharness_runner.html")
         self.logger.debug("Loading %s" % url)
         try:
@@ -380,17 +374,21 @@ class MarionetteRefTestExecutor(RefTestE
             self.protocol.marionette.execute_script(self.script)
             self.protocol.marionette.switch_to_window(self.protocol.marionette.window_handles[-1])
             self.has_window = True
 
         result = self.implementation.run_test(test)
 
         return self.convert_result(test, result)
 
-    def screenshot(self, test):
+    def screenshot(self, test, viewport_size, dpi):
+        # https://github.com/w3c/wptrunner/issues/166
+        assert viewport_size is None
+        assert dpi is None
+
         timeout =  self.timeout_multiplier * test.timeout if self.debug_info is None else None
 
         test_url = self.test_url(test)
 
         return MarionetteRun(self.logger,
                              self._screenshot,
                              self.protocol.marionette,
                              test_url,
--- a/testing/web-platform/harness/wptrunner/executors/executorselenium.py
+++ b/testing/web-platform/harness/wptrunner/executors/executorselenium.py
@@ -242,17 +242,21 @@ class SeleniumRefTestExecutor(RefTestExe
             self.protocol.webdriver.switch_to_window(
                 self.protocol.webdriver.window_handles[-1])
             self.has_window = True
 
         result = self.implementation.run_test(test)
 
         return self.convert_result(test, result)
 
-    def screenshot(self, test):
+    def screenshot(self, test, viewport_size, dpi):
+        # https://github.com/w3c/wptrunner/issues/166
+        assert viewport_size is None
+        assert dpi is None
+
         return SeleniumRun(self._screenshot,
                            self.protocol.webdriver,
                            self.test_url(test),
                            test.timeout).run()
 
     def _screenshot(self, webdriver, url, timeout):
         webdriver.get(url)
 
--- a/testing/web-platform/harness/wptrunner/executors/executorservo.py
+++ b/testing/web-platform/harness/wptrunner/executors/executorservo.py
@@ -191,33 +191,39 @@ class ServoRefTestExecutor(ProcessTestEx
     def teardown(self):
         try:
             os.unlink(self.hosts_path)
         except OSError:
             pass
         os.rmdir(self.tempdir)
         ProcessTestExecutor.teardown(self)
 
-    def screenshot(self, test):
+    def screenshot(self, test, viewport_size, dpi):
         full_url = self.test_url(test)
 
         with TempFilename(self.tempdir) as output_path:
             debug_args, command = browser_command(
                 self.binary,
                 [render_arg(self.browser.render_backend), "--hard-fail", "--exit",
-                 "-u", "Servo/wptrunner", "-Z", "disable-text-aa",
+                 "-u", "Servo/wptrunner", "-Z", "disable-text-aa,load-webfonts-synchronously",
                  "--output=%s" % output_path, full_url],
                 self.debug_info)
 
             for stylesheet in self.browser.user_stylesheets:
                 command += ["--user-stylesheet", stylesheet]
 
             for pref in test.environment.get('prefs', {}):
                 command += ["--pref", pref]
 
+            if viewport_size:
+                command += ["--resolution", viewport_size]
+
+            if dpi:
+                command += ["--device-pixel-ratio", dpi]
+
             self.command = debug_args + command
 
             env = os.environ.copy()
             env["HOST_FILE"] = self.hosts_path
             env["RUST_BACKTRACE"] = "1"
 
             if not self.interactive:
                 self.proc = ProcessHandler(self.command,
--- a/testing/web-platform/harness/wptrunner/executors/executorservodriver.py
+++ b/testing/web-platform/harness/wptrunner/executors/executorservodriver.py
@@ -221,17 +221,21 @@ class ServoWebDriverRefTestExecutor(RefT
             return test.result_cls("TIMEOUT", None), []
         except Exception as e:
             message = getattr(e, "message", "")
             if message:
                 message += "\n"
             message += traceback.format_exc(e)
             return test.result_cls("ERROR", message), []
 
-    def screenshot(self, test):
+    def screenshot(self, test, viewport_size, dpi):
+        # https://github.com/w3c/wptrunner/issues/166
+        assert viewport_size is None
+        assert dpi is None
+
         timeout = (test.timeout * self.timeout_multiplier + extra_timeout
                    if self.debug_info is None else None)
 
         if self.timeout != timeout:
             try:
                 self.protocol.session.timeouts.script = timeout
                 self.timeout = timeout
             except IOError:
--- a/testing/web-platform/harness/wptrunner/manifestexpected.py
+++ b/testing/web-platform/harness/wptrunner/manifestexpected.py
@@ -24,20 +24,20 @@ def data_cls_getter(output_node, visited
         return ExpectedManifest
     if isinstance(output_node, ExpectedManifest):
         return TestNode
     if isinstance(output_node, TestNode):
         return SubtestNode
     raise ValueError
 
 
-def bool_prop(name, node):
-    """Boolean property"""
+def disabled(node):
+    """Boolean indicating whether the test is disabled"""
     try:
-        return node.get(name)
+        return node.get("disabled")
     except KeyError:
         return None
 
 
 def tags(node):
     """Set of tags that have been applied to the test"""
     try:
         value = node.get("tags")
@@ -104,39 +104,31 @@ class ExpectedManifest(ManifestItem):
 
     @property
     def url(self):
         return urlparse.urljoin(self.url_base,
                                 "/".join(self.test_path.split(os.path.sep)))
 
     @property
     def disabled(self):
-        return bool_prop("disabled", self)
-
-    @property
-    def restart_after(self):
-        return bool_prop("restart-after", self)
+        return disabled(self)
 
     @property
     def tags(self):
         return tags(self)
 
     @property
     def prefs(self):
         return prefs(self)
 
 
 class DirectoryManifest(ManifestItem):
     @property
     def disabled(self):
-        return bool_prop("disabled", self)
-
-    @property
-    def restart_after(self):
-        return bool_prop("restart-after", self)
+        return disabled(self)
 
     @property
     def tags(self):
         return tags(self)
 
     @property
     def prefs(self):
         return prefs(self)
@@ -167,21 +159,17 @@ class TestNode(ManifestItem):
         return self.get("type")
 
     @property
     def id(self):
         return urlparse.urljoin(self.parent.url, self.name)
 
     @property
     def disabled(self):
-        return bool_prop("disabled", self)
-
-    @property
-    def restart_after(self):
-        return bool_prop("restart-after", self)
+        return disabled(self)
 
     @property
     def tags(self):
         return tags(self)
 
     @property
     def prefs(self):
         return prefs(self)
--- a/testing/web-platform/harness/wptrunner/testrunner.py
+++ b/testing/web-platform/harness/wptrunner/testrunner.py
@@ -519,18 +519,17 @@ class TestRunnerManager(threading.Thread
         self.logger.test_end(test.id,
                              status,
                              message=file_result.message,
                              expected=expected,
                              extra=file_result.extra)
 
         self.test = None
 
-        restart_before_next = (test.restart_after or
-                               file_result.status in ("CRASH", "EXTERNAL-TIMEOUT") or
+        restart_before_next = (file_result.status in ("CRASH", "EXTERNAL-TIMEOUT") or
                                subtest_unexpected or is_unexpected)
 
         if (self.pause_after_test or
             (self.pause_on_unexpected and (subtest_unexpected or is_unexpected))):
             self.logger.info("Pausing until the browser exits")
             self.send_message("wait")
         else:
             self.after_test_ended(restart_before_next)
--- a/testing/web-platform/harness/wptrunner/wptcommandline.py
+++ b/testing/web-platform/harness/wptrunner/wptcommandline.py
@@ -67,16 +67,18 @@ def create_parser(product_choices=None):
                         help="Split run into groups by directories. With a parameter,"
                         "limit the depth of splits e.g. --run-by-dir=1 to split by top-level"
                         "directory")
 
     parser.add_argument("--timeout-multiplier", action="store", type=float, default=None,
                         help="Multiplier relative to standard test timeout to use")
     parser.add_argument("--repeat", action="store", type=int, default=1,
                         help="Number of times to run the tests")
+    parser.add_argument("--repeat-until-unexpected", action="store_true", default=None,
+                        help="Run tests in a loop until one returns an unexpected result")
 
     parser.add_argument("--no-capture-stdio", action="store_true", default=False,
                         help="Don't capture stdio and write to logging")
 
     parser.add_argument("--product", action="store", choices=product_choices,
                         default=None, help="Browser against which to run tests")
 
     parser.add_argument("--list-test-groups", action="store_true",
@@ -121,16 +123,19 @@ def create_parser(product_choices=None):
     debugging_group.add_argument('--pause-on-unexpected', action="store_true",
                                  help="Halt the test runner when an unexpected result is encountered")
 
     debugging_group.add_argument("--symbols-path", action="store", type=url_or_path,
                                  help="Path or url to symbols file used to analyse crash minidumps.")
     debugging_group.add_argument("--stackwalk-binary", action="store", type=abs_path,
                                  help="Path to stackwalker program used to analyse minidumps.")
 
+    debugging_group.add_argument("--pdb", action="store_true",
+                                 help="Drop into pdb on python exception")
+
     chunking_group = parser.add_argument_group("Test Chunking")
     chunking_group.add_argument("--total-chunks", action="store", type=int, default=1,
                                 help="Total number of chunks to use")
     chunking_group.add_argument("--this-chunk", action="store", type=int, default=1,
                                 help="Chunk number to run")
     chunking_group.add_argument("--chunk-type", action="store", choices=["none", "equal_time", "hash"],
                                 default=None, help="Chunking type to use")
 
--- a/testing/web-platform/harness/wptrunner/wptrunner.py
+++ b/testing/web-platform/harness/wptrunner/wptrunner.py
@@ -97,16 +97,18 @@ def list_disabled(test_paths, product, *
         for test in tests:
             rv.append({"test": test.id, "reason": test.disabled()})
     print json.dumps(rv, indent=2)
 
 
 def get_pause_after_test(test_loader, **kwargs):
     total_tests = sum(len(item) for item in test_loader.tests.itervalues())
     if kwargs["pause_after_test"] is None:
+        if kwargs["repeat_until_unexpected"]:
+            return False
         if kwargs["repeat"] == 1 and total_tests == 1:
             return True
         return False
     return kwargs["pause_after_test"]
 
 
 def run_tests(config, test_paths, product, **kwargs):
     with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]):
@@ -155,20 +157,25 @@ def run_tests(config, test_paths, produc
                 test_environment.ensure_started()
             except env.TestEnvironmentError as e:
                 logger.critical("Error starting test environment: %s" % e.message)
                 raise
 
             browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs)
 
             repeat = kwargs["repeat"]
-            for repeat_count in xrange(repeat):
-                if repeat > 1:
-                    logger.info("Repetition %i / %i" % (repeat_count + 1, repeat))
+            repeat_count = 0
+            repeat_until_unexpected = kwargs["repeat_until_unexpected"]
 
+            while repeat_count < repeat or repeat_until_unexpected:
+                repeat_count += 1
+                if repeat_until_unexpected:
+                    logger.info("Repetition %i" % (repeat_count))
+                elif repeat > 1:
+                    logger.info("Repetition %i / %i" % (repeat_count, repeat))
 
                 unexpected_count = 0
                 logger.suite_start(test_loader.test_ids, run_info)
                 for test_type in kwargs["test_types"]:
                     logger.info("Running %s tests" % test_type)
 
                     for test in test_loader.disabled_tests[test_type]:
                         logger.test_start(test.id)
@@ -203,33 +210,38 @@ def run_tests(config, test_paths, produc
                         except KeyboardInterrupt:
                             logger.critical("Main thread got signal")
                             manager_group.stop()
                             raise
                     unexpected_count += manager_group.unexpected_count()
 
                 unexpected_total += unexpected_count
                 logger.info("Got %i unexpected results" % unexpected_count)
+                if repeat_until_unexpected and unexpected_total > 0:
+                    break
                 logger.suite_end()
 
     return unexpected_total == 0
 
 
 def main():
     """Main entry point when calling from the command line"""
+    kwargs = wptcommandline.parse_args()
+
     try:
-        kwargs = wptcommandline.parse_args()
-
         if kwargs["prefs_root"] is None:
             kwargs["prefs_root"] = os.path.abspath(os.path.join(here, "prefs"))
 
         setup_logging(kwargs, {"raw": sys.stdout})
 
         if kwargs["list_test_groups"]:
             list_test_groups(**kwargs)
         elif kwargs["list_disabled"]:
             list_disabled(**kwargs)
         else:
             return not run_tests(**kwargs)
     except Exception:
-        import pdb, traceback
-        print traceback.format_exc()
-        pdb.post_mortem()
+        if kwargs["pdb"]:
+            import pdb, traceback
+            print traceback.format_exc()
+            pdb.post_mortem()
+        else:
+            raise
--- a/testing/web-platform/harness/wptrunner/wpttest.py
+++ b/testing/web-platform/harness/wptrunner/wpttest.py
@@ -145,24 +145,16 @@ class Test(object):
     def disabled(self, subtest=None):
         for meta in self.itermeta(subtest):
             disabled = meta.disabled
             if disabled is not None:
                 return disabled
         return None
 
     @property
-    def restart_after(self):
-        for meta in self.itermeta(None):
-            restart_after = meta.restart_after
-            if restart_after is not None:
-                return True
-        return False
-
-    @property
     def tags(self):
         tags = set()
         for meta in self.itermeta():
             meta_tags = meta.tags
             if atom_reset in meta_tags:
                 tags = meta_tags.copy()
                 tags.remove(atom_reset)
             else:
@@ -217,24 +209,28 @@ class ManualTest(Test):
     def id(self):
         return self.url
 
 
 class ReftestTest(Test):
     result_cls = ReftestResult
     test_type = "reftest"
 
-    def __init__(self, url, inherit_metadata, test_metadata, references, timeout=DEFAULT_TIMEOUT, path=None, protocol="http"):
+    def __init__(self, url, inherit_metadata, test_metadata, references,
+                 timeout=DEFAULT_TIMEOUT, path=None, viewport_size=None,
+                 dpi=None, protocol="http"):
         Test.__init__(self, url, inherit_metadata, test_metadata, timeout, path, protocol)
 
         for _, ref_type in references:
             if ref_type not in ("==", "!="):
                 raise ValueError
 
         self.references = references
+        self.viewport_size = viewport_size
+        self.dpi = dpi
 
     @classmethod
     def from_manifest(cls,
                       manifest_test,
                       inherit_metadata,
                       test_metadata,
                       nodes=None,
                       references_seen=None):
@@ -249,16 +245,18 @@ class ReftestTest(Test):
         url = manifest_test.url
 
         node = cls(manifest_test.url,
                    inherit_metadata,
                    test_metadata,
                    [],
                    timeout=timeout,
                    path=manifest_test.path,
+                   viewport_size=manifest_test.viewport_size,
+                   dpi=manifest_test.dpi,
                    protocol="https" if hasattr(manifest_test, "https") and manifest_test.https else "http")
 
         nodes[url] = node
 
         for ref_url, ref_type in manifest_test.references:
             comparison_key = (ref_type,) + tuple(sorted([url, ref_url]))
             if ref_url in nodes:
                 manifest_node = ref_url