Bug 1257799 - Update to latest wptrunner, a=testonly
authorJames Graham <james@hoppipolla.co.uk>
Wed, 16 Mar 2016 13:55:15 +0000
changeset 289398 8938d99b75bb12ce23edc53f90c87b2d7ba1071d
parent 289397 6ab07b5c4c3eab8e76793e6ff630a94b59442991
child 289399 39ab55ac2fc8017c986dcf8629129379f2e9351b
push id19656
push usergwagner@mozilla.com
push dateMon, 04 Apr 2016 13:43:23 +0000
treeherderb2g-inbound@e99061fde28a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstestonly
bugs1257799
milestone48.0a1
Bug 1257799 - Update to latest wptrunner, a=testonly MozReview-Commit-ID: K7Ro7yG0Nds
testing/web-platform/harness/docs/expectation.rst
testing/web-platform/harness/wptrunner/browsers/firefox.py
testing/web-platform/harness/wptrunner/executors/base.py
testing/web-platform/harness/wptrunner/executors/executormarionette.py
testing/web-platform/harness/wptrunner/executors/executorselenium.py
testing/web-platform/harness/wptrunner/executors/executorservo.py
testing/web-platform/harness/wptrunner/executors/executorservodriver.py
testing/web-platform/harness/wptrunner/manifestexpected.py
testing/web-platform/harness/wptrunner/testrunner.py
testing/web-platform/harness/wptrunner/wptcommandline.py
testing/web-platform/harness/wptrunner/wptrunner.py
testing/web-platform/harness/wptrunner/wpttest.py
--- a/testing/web-platform/harness/docs/expectation.rst
+++ b/testing/web-platform/harness/docs/expectation.rst
@@ -198,20 +198,16 @@ When used for expectation data, manifest
    reference.
 
  * A key ``expected`` giving the expectation value of each (sub)test.
 
  * A key ``disabled`` which can be set to any value to indicate that
    the (sub)test is disabled and should either not be run (for tests)
    or that its results should be ignored (subtests).
 
- * A key ``restart-after`` which can be set to any value to indicate that
-   the runner should restart the browser after running this test (e.g. to
-   clear out unwanted state).
-
  * Variables ``debug``, ``os``, ``version``, ``processor`` and
    ``bits`` that describe the configuration of the browser under
    test. ``debug`` is a boolean indicating whether a build is a debug
    build. ``os`` is a string indicating the operating system, and
    ``version`` a string indicating the particular version of that
    operating system. ``processor`` is a string indicating the
    processor architecture and ``bits`` an integer indicating the
    number of bits. This information is typically provided by
--- a/testing/web-platform/harness/wptrunner/browsers/firefox.py
+++ b/testing/web-platform/harness/wptrunner/browsers/firefox.py
@@ -77,16 +77,17 @@ def run_info_extras(**kwargs):
     return {"e10s": kwargs["gecko_e10s"]}
 
 
 def update_properties():
     return ["debug", "e10s", "os", "version", "processor", "bits"], {"debug", "e10s"}
 
 class FirefoxBrowser(Browser):
     used_ports = set()
+    init_timeout = 60
 
     def __init__(self, logger, binary, prefs_root, debug_info=None,
                  symbols_path=None, stackwalk_binary=None, certutil_binary=None,
                  ca_certificate_path=None, e10s=False):
         Browser.__init__(self, logger)
         self.binary = binary
         self.prefs_root = prefs_root
         self.marionette_port = None
--- a/testing/web-platform/harness/wptrunner/executors/base.py
+++ b/testing/web-platform/harness/wptrunner/executors/base.py
@@ -201,92 +201,96 @@ class RefTestImplementation(object):
         # retrieve the screenshot from the cache directly in the future
         self.screenshot_cache = self.executor.screenshot_cache
         self.message = None
 
     @property
     def logger(self):
         return self.executor.logger
 
-    def get_hash(self, test):
+    def get_hash(self, test, viewport_size, dpi):
         timeout = test.timeout * self.timeout_multiplier
+        key = (test.url, viewport_size, dpi)
 
-        if test.url not in self.screenshot_cache:
-            success, data = self.executor.screenshot(test)
+        if key not in self.screenshot_cache:
+            success, data = self.executor.screenshot(test, viewport_size, dpi)
 
             if not success:
                 return False, data
 
             screenshot = data
             hash_value = hashlib.sha1(screenshot).hexdigest()
 
-            self.screenshot_cache[test.url] = (hash_value, None)
+            self.screenshot_cache[key] = (hash_value, None)
 
-            rv = True, (hash_value, screenshot)
+            rv = (hash_value, screenshot)
         else:
-            rv = True, self.screenshot_cache[test.url]
+            rv = self.screenshot_cache[key]
 
-        self.message.append("%s %s" % (test.url, rv[1][0]))
-        return rv
+        self.message.append("%s %s" % (test.url, rv[0]))
+        return True, rv
 
     def is_pass(self, lhs_hash, rhs_hash, relation):
         assert relation in ("==", "!=")
         self.message.append("Testing %s %s %s" % (lhs_hash, relation, rhs_hash))
         return ((relation == "==" and lhs_hash == rhs_hash) or
                 (relation == "!=" and lhs_hash != rhs_hash))
 
     def run_test(self, test):
+        viewport_size = test.viewport_size
+        dpi = test.dpi
         self.message = []
 
         # Depth-first search of reference tree, with the goal
         # of reachings a leaf node with only pass results
 
         stack = list(((test, item[0]), item[1]) for item in reversed(test.references))
         while stack:
             hashes = [None, None]
             screenshots = [None, None]
 
             nodes, relation = stack.pop()
 
             for i, node in enumerate(nodes):
-                success, data = self.get_hash(node)
+                success, data = self.get_hash(node, viewport_size, dpi)
                 if success is False:
                     return {"status": data[0], "message": data[1]}
 
                 hashes[i], screenshots[i] = data
 
             if self.is_pass(hashes[0], hashes[1], relation):
                 if nodes[1].references:
                     stack.extend(list(((nodes[1], item[0]), item[1]) for item in reversed(nodes[1].references)))
                 else:
                     # We passed
                     return {"status":"PASS", "message": None}
 
         # We failed, so construct a failure message
 
         for i, (node, screenshot) in enumerate(zip(nodes, screenshots)):
             if screenshot is None:
-                success, screenshot = self.retake_screenshot(node)
+                success, screenshot = self.retake_screenshot(node, viewport_size, dpi)
                 if success:
                     screenshots[i] = screenshot
 
         log_data = [{"url": nodes[0].url, "screenshot": screenshots[0]}, relation,
                     {"url": nodes[1].url, "screenshot": screenshots[1]}]
 
         return {"status": "FAIL",
                 "message": "\n".join(self.message),
                 "extra": {"reftest_screenshots": log_data}}
 
-    def retake_screenshot(self, node):
-        success, data = self.executor.screenshot(node)
+    def retake_screenshot(self, node, viewport_size, dpi):
+        success, data = self.executor.screenshot(node, viewport_size, dpi)
         if not success:
             return False, data
 
-        hash_val, _ = self.screenshot_cache[node.url]
-        self.screenshot_cache[node.url] = hash_val, data
+        key = (node.url, viewport_size, dpi)
+        hash_val, _ = self.screenshot_cache[key]
+        self.screenshot_cache[key] = hash_val, data
         return True, data
 
 class Protocol(object):
     def __init__(self, executor, browser):
         self.executor = executor
         self.browser = browser
 
     @property
--- a/testing/web-platform/harness/wptrunner/executors/executormarionette.py
+++ b/testing/web-platform/harness/wptrunner/executors/executormarionette.py
@@ -102,22 +102,16 @@ class MarionetteProtocol(Protocol):
         try:
             # Get a simple property over the connection
             self.marionette.current_window_handle
         except Exception:
             return False
         return True
 
     def after_connect(self):
-        # Turn off debug-level logging by default since this is so verbose
-        with self.marionette.using_context("chrome"):
-            self.marionette.execute_script("""
-              Components.utils.import("resource://gre/modules/Log.jsm");
-              Log.repository.getLogger("Marionette").level = Log.Level.Info;
-            """)
         self.load_runner("http")
 
     def load_runner(self, protocol):
         # Check if we previously had a test window open, and if we did make sure it's closed
         self.marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
         url = urlparse.urljoin(self.executor.server_url(protocol), "/testharness_runner.html")
         self.logger.debug("Loading %s" % url)
         try:
@@ -380,17 +374,21 @@ class MarionetteRefTestExecutor(RefTestE
             self.protocol.marionette.execute_script(self.script)
             self.protocol.marionette.switch_to_window(self.protocol.marionette.window_handles[-1])
             self.has_window = True
 
         result = self.implementation.run_test(test)
 
         return self.convert_result(test, result)
 
-    def screenshot(self, test):
+    def screenshot(self, test, viewport_size, dpi):
+        # https://github.com/w3c/wptrunner/issues/166
+        assert viewport_size is None
+        assert dpi is None
+
         timeout =  self.timeout_multiplier * test.timeout if self.debug_info is None else None
 
         test_url = self.test_url(test)
 
         return MarionetteRun(self.logger,
                              self._screenshot,
                              self.protocol.marionette,
                              test_url,
--- a/testing/web-platform/harness/wptrunner/executors/executorselenium.py
+++ b/testing/web-platform/harness/wptrunner/executors/executorselenium.py
@@ -242,17 +242,21 @@ class SeleniumRefTestExecutor(RefTestExe
             self.protocol.webdriver.switch_to_window(
                 self.protocol.webdriver.window_handles[-1])
             self.has_window = True
 
         result = self.implementation.run_test(test)
 
         return self.convert_result(test, result)
 
-    def screenshot(self, test):
+    def screenshot(self, test, viewport_size, dpi):
+        # https://github.com/w3c/wptrunner/issues/166
+        assert viewport_size is None
+        assert dpi is None
+
         return SeleniumRun(self._screenshot,
                            self.protocol.webdriver,
                            self.test_url(test),
                            test.timeout).run()
 
     def _screenshot(self, webdriver, url, timeout):
         webdriver.get(url)
 
--- a/testing/web-platform/harness/wptrunner/executors/executorservo.py
+++ b/testing/web-platform/harness/wptrunner/executors/executorservo.py
@@ -191,33 +191,39 @@ class ServoRefTestExecutor(ProcessTestEx
     def teardown(self):
         try:
             os.unlink(self.hosts_path)
         except OSError:
             pass
         os.rmdir(self.tempdir)
         ProcessTestExecutor.teardown(self)
 
-    def screenshot(self, test):
+    def screenshot(self, test, viewport_size, dpi):
         full_url = self.test_url(test)
 
         with TempFilename(self.tempdir) as output_path:
             debug_args, command = browser_command(
                 self.binary,
                 [render_arg(self.browser.render_backend), "--hard-fail", "--exit",
-                 "-u", "Servo/wptrunner", "-Z", "disable-text-aa",
+                 "-u", "Servo/wptrunner", "-Z", "disable-text-aa,load-webfonts-synchronously",
                  "--output=%s" % output_path, full_url],
                 self.debug_info)
 
             for stylesheet in self.browser.user_stylesheets:
                 command += ["--user-stylesheet", stylesheet]
 
             for pref in test.environment.get('prefs', {}):
                 command += ["--pref", pref]
 
+            if viewport_size:
+                command += ["--resolution", viewport_size]
+
+            if dpi:
+                command += ["--device-pixel-ratio", dpi]
+
             self.command = debug_args + command
 
             env = os.environ.copy()
             env["HOST_FILE"] = self.hosts_path
             env["RUST_BACKTRACE"] = "1"
 
             if not self.interactive:
                 self.proc = ProcessHandler(self.command,
--- a/testing/web-platform/harness/wptrunner/executors/executorservodriver.py
+++ b/testing/web-platform/harness/wptrunner/executors/executorservodriver.py
@@ -221,17 +221,21 @@ class ServoWebDriverRefTestExecutor(RefT
             return test.result_cls("TIMEOUT", None), []
         except Exception as e:
             message = getattr(e, "message", "")
             if message:
                 message += "\n"
             message += traceback.format_exc(e)
             return test.result_cls("ERROR", message), []
 
-    def screenshot(self, test):
+    def screenshot(self, test, viewport_size, dpi):
+        # https://github.com/w3c/wptrunner/issues/166
+        assert viewport_size is None
+        assert dpi is None
+
         timeout = (test.timeout * self.timeout_multiplier + extra_timeout
                    if self.debug_info is None else None)
 
         if self.timeout != timeout:
             try:
                 self.protocol.session.timeouts.script = timeout
                 self.timeout = timeout
             except IOError:
--- a/testing/web-platform/harness/wptrunner/manifestexpected.py
+++ b/testing/web-platform/harness/wptrunner/manifestexpected.py
@@ -24,20 +24,20 @@ def data_cls_getter(output_node, visited
         return ExpectedManifest
     if isinstance(output_node, ExpectedManifest):
         return TestNode
     if isinstance(output_node, TestNode):
         return SubtestNode
     raise ValueError
 
 
-def bool_prop(name, node):
-    """Boolean property"""
+def disabled(node):
+    """Boolean indicating whether the test is disabled"""
     try:
-        return node.get(name)
+        return node.get("disabled")
     except KeyError:
         return None
 
 
 def tags(node):
     """Set of tags that have been applied to the test"""
     try:
         value = node.get("tags")
@@ -104,39 +104,31 @@ class ExpectedManifest(ManifestItem):
 
     @property
     def url(self):
         return urlparse.urljoin(self.url_base,
                                 "/".join(self.test_path.split(os.path.sep)))
 
     @property
     def disabled(self):
-        return bool_prop("disabled", self)
-
-    @property
-    def restart_after(self):
-        return bool_prop("restart-after", self)
+        return disabled(self)
 
     @property
     def tags(self):
         return tags(self)
 
     @property
     def prefs(self):
         return prefs(self)
 
 
 class DirectoryManifest(ManifestItem):
     @property
     def disabled(self):
-        return bool_prop("disabled", self)
-
-    @property
-    def restart_after(self):
-        return bool_prop("restart-after", self)
+        return disabled(self)
 
     @property
     def tags(self):
         return tags(self)
 
     @property
     def prefs(self):
         return prefs(self)
@@ -167,21 +159,17 @@ class TestNode(ManifestItem):
         return self.get("type")
 
     @property
     def id(self):
         return urlparse.urljoin(self.parent.url, self.name)
 
     @property
     def disabled(self):
-        return bool_prop("disabled", self)
-
-    @property
-    def restart_after(self):
-        return bool_prop("restart-after", self)
+        return disabled(self)
 
     @property
     def tags(self):
         return tags(self)
 
     @property
     def prefs(self):
         return prefs(self)
--- a/testing/web-platform/harness/wptrunner/testrunner.py
+++ b/testing/web-platform/harness/wptrunner/testrunner.py
@@ -519,18 +519,17 @@ class TestRunnerManager(threading.Thread
         self.logger.test_end(test.id,
                              status,
                              message=file_result.message,
                              expected=expected,
                              extra=file_result.extra)
 
         self.test = None
 
-        restart_before_next = (test.restart_after or
-                               file_result.status in ("CRASH", "EXTERNAL-TIMEOUT") or
+        restart_before_next = (file_result.status in ("CRASH", "EXTERNAL-TIMEOUT") or
                                subtest_unexpected or is_unexpected)
 
         if (self.pause_after_test or
             (self.pause_on_unexpected and (subtest_unexpected or is_unexpected))):
             self.logger.info("Pausing until the browser exits")
             self.send_message("wait")
         else:
             self.after_test_ended(restart_before_next)
--- a/testing/web-platform/harness/wptrunner/wptcommandline.py
+++ b/testing/web-platform/harness/wptrunner/wptcommandline.py
@@ -67,16 +67,18 @@ def create_parser(product_choices=None):
                         help="Split run into groups by directories. With a parameter,"
                         "limit the depth of splits e.g. --run-by-dir=1 to split by top-level"
                         "directory")
 
     parser.add_argument("--timeout-multiplier", action="store", type=float, default=None,
                         help="Multiplier relative to standard test timeout to use")
     parser.add_argument("--repeat", action="store", type=int, default=1,
                         help="Number of times to run the tests")
+    parser.add_argument("--repeat-until-unexpected", action="store_true", default=None,
+                        help="Run tests in a loop until one returns an unexpected result")
 
     parser.add_argument("--no-capture-stdio", action="store_true", default=False,
                         help="Don't capture stdio and write to logging")
 
     parser.add_argument("--product", action="store", choices=product_choices,
                         default=None, help="Browser against which to run tests")
 
     parser.add_argument("--list-test-groups", action="store_true",
@@ -121,16 +123,19 @@ def create_parser(product_choices=None):
     debugging_group.add_argument('--pause-on-unexpected', action="store_true",
                                  help="Halt the test runner when an unexpected result is encountered")
 
     debugging_group.add_argument("--symbols-path", action="store", type=url_or_path,
                                  help="Path or url to symbols file used to analyse crash minidumps.")
     debugging_group.add_argument("--stackwalk-binary", action="store", type=abs_path,
                                  help="Path to stackwalker program used to analyse minidumps.")
 
+    debugging_group.add_argument("--pdb", action="store_true",
+                                 help="Drop into pdb on python exception")
+
     chunking_group = parser.add_argument_group("Test Chunking")
     chunking_group.add_argument("--total-chunks", action="store", type=int, default=1,
                                 help="Total number of chunks to use")
     chunking_group.add_argument("--this-chunk", action="store", type=int, default=1,
                                 help="Chunk number to run")
     chunking_group.add_argument("--chunk-type", action="store", choices=["none", "equal_time", "hash"],
                                 default=None, help="Chunking type to use")
 
--- a/testing/web-platform/harness/wptrunner/wptrunner.py
+++ b/testing/web-platform/harness/wptrunner/wptrunner.py
@@ -97,16 +97,18 @@ def list_disabled(test_paths, product, *
         for test in tests:
             rv.append({"test": test.id, "reason": test.disabled()})
     print json.dumps(rv, indent=2)
 
 
 def get_pause_after_test(test_loader, **kwargs):
     total_tests = sum(len(item) for item in test_loader.tests.itervalues())
     if kwargs["pause_after_test"] is None:
+        if kwargs["repeat_until_unexpected"]:
+            return False
         if kwargs["repeat"] == 1 and total_tests == 1:
             return True
         return False
     return kwargs["pause_after_test"]
 
 
 def run_tests(config, test_paths, product, **kwargs):
     with wptlogging.CaptureIO(logger, not kwargs["no_capture_stdio"]):
@@ -155,20 +157,25 @@ def run_tests(config, test_paths, produc
                 test_environment.ensure_started()
             except env.TestEnvironmentError as e:
                 logger.critical("Error starting test environment: %s" % e.message)
                 raise
 
             browser_kwargs = get_browser_kwargs(ssl_env=ssl_env, **kwargs)
 
             repeat = kwargs["repeat"]
-            for repeat_count in xrange(repeat):
-                if repeat > 1:
-                    logger.info("Repetition %i / %i" % (repeat_count + 1, repeat))
+            repeat_count = 0
+            repeat_until_unexpected = kwargs["repeat_until_unexpected"]
 
+            while repeat_count < repeat or repeat_until_unexpected:
+                repeat_count += 1
+                if repeat_until_unexpected:
+                    logger.info("Repetition %i" % (repeat_count))
+                elif repeat > 1:
+                    logger.info("Repetition %i / %i" % (repeat_count, repeat))
 
                 unexpected_count = 0
                 logger.suite_start(test_loader.test_ids, run_info)
                 for test_type in kwargs["test_types"]:
                     logger.info("Running %s tests" % test_type)
 
                     for test in test_loader.disabled_tests[test_type]:
                         logger.test_start(test.id)
@@ -203,33 +210,38 @@ def run_tests(config, test_paths, produc
                         except KeyboardInterrupt:
                             logger.critical("Main thread got signal")
                             manager_group.stop()
                             raise
                     unexpected_count += manager_group.unexpected_count()
 
                 unexpected_total += unexpected_count
                 logger.info("Got %i unexpected results" % unexpected_count)
+                if repeat_until_unexpected and unexpected_total > 0:
+                    break
                 logger.suite_end()
 
     return unexpected_total == 0
 
 
 def main():
     """Main entry point when calling from the command line"""
+    kwargs = wptcommandline.parse_args()
+
     try:
-        kwargs = wptcommandline.parse_args()
-
         if kwargs["prefs_root"] is None:
             kwargs["prefs_root"] = os.path.abspath(os.path.join(here, "prefs"))
 
         setup_logging(kwargs, {"raw": sys.stdout})
 
         if kwargs["list_test_groups"]:
             list_test_groups(**kwargs)
         elif kwargs["list_disabled"]:
             list_disabled(**kwargs)
         else:
             return not run_tests(**kwargs)
     except Exception:
-        import pdb, traceback
-        print traceback.format_exc()
-        pdb.post_mortem()
+        if kwargs["pdb"]:
+            import pdb, traceback
+            print traceback.format_exc()
+            pdb.post_mortem()
+        else:
+            raise
--- a/testing/web-platform/harness/wptrunner/wpttest.py
+++ b/testing/web-platform/harness/wptrunner/wpttest.py
@@ -145,24 +145,16 @@ class Test(object):
     def disabled(self, subtest=None):
         for meta in self.itermeta(subtest):
             disabled = meta.disabled
             if disabled is not None:
                 return disabled
         return None
 
     @property
-    def restart_after(self):
-        for meta in self.itermeta(None):
-            restart_after = meta.restart_after
-            if restart_after is not None:
-                return True
-        return False
-
-    @property
     def tags(self):
         tags = set()
         for meta in self.itermeta():
             meta_tags = meta.tags
             if atom_reset in meta_tags:
                 tags = meta_tags.copy()
                 tags.remove(atom_reset)
             else:
@@ -217,24 +209,28 @@ class ManualTest(Test):
     def id(self):
         return self.url
 
 
 class ReftestTest(Test):
     result_cls = ReftestResult
     test_type = "reftest"
 
-    def __init__(self, url, inherit_metadata, test_metadata, references, timeout=DEFAULT_TIMEOUT, path=None, protocol="http"):
+    def __init__(self, url, inherit_metadata, test_metadata, references,
+                 timeout=DEFAULT_TIMEOUT, path=None, viewport_size=None,
+                 dpi=None, protocol="http"):
         Test.__init__(self, url, inherit_metadata, test_metadata, timeout, path, protocol)
 
         for _, ref_type in references:
             if ref_type not in ("==", "!="):
                 raise ValueError
 
         self.references = references
+        self.viewport_size = viewport_size
+        self.dpi = dpi
 
     @classmethod
     def from_manifest(cls,
                       manifest_test,
                       inherit_metadata,
                       test_metadata,
                       nodes=None,
                       references_seen=None):
@@ -249,16 +245,18 @@ class ReftestTest(Test):
         url = manifest_test.url
 
         node = cls(manifest_test.url,
                    inherit_metadata,
                    test_metadata,
                    [],
                    timeout=timeout,
                    path=manifest_test.path,
+                   viewport_size=manifest_test.viewport_size,
+                   dpi=manifest_test.dpi,
                    protocol="https" if hasattr(manifest_test, "https") and manifest_test.https else "http")
 
         nodes[url] = node
 
         for ref_url, ref_type in manifest_test.references:
             comparison_key = (ref_type,) + tuple(sorted([url, ref_url]))
             if ref_url in nodes:
                 manifest_node = ref_url