Bug 1505474 - Use downloaded manifest for wpt in jsshell, r=bbouvier,Ms2ger
authorJames Graham <james@hoppipolla.co.uk>
Fri, 16 Nov 2018 15:23:21 +0000
changeset 446810 32aa5695f2ca06124214d447fda352709ea2736c
parent 446809 f38aa27974eea413de93123272629af54443ec0b
child 446811 aa24577bdeae449b6b51f1184cc6aec809fc3659
push id35052
push userapavel@mozilla.com
push dateSat, 17 Nov 2018 11:25:40 +0000
treeherdermozilla-central@efc1da42132b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbbouvier, Ms2ger
bugs1505474, 1497898
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1505474 - Use downloaded manifest for wpt in jsshell, r=bbouvier,Ms2ger web-platform-tests gets its test list from a manifest file that's generated from the content of the tests. Unfortunately generating this manifest is slow, so it's unreasonable to create it from scratch for every test run. Until recently the generated manifest was kept in-tree, which was suboptimal in a few ways: * The manifest tended to get out of sync with the actual source * The large json file caused problems for tooling including source control and the review frontends. We previously switched `mach wpt` to download a manifest on demand and apply an incremental update. However this work missed the usage in jstests.py. This continued to use the increasing outdated in-tree manifest, which causes a number of problems * It doesn't have an up-to-date list of tests * It blocks removing that file * It blocks landing various optimisations to make updating the manifest faster. This patch fixes jstests.py to use a downloaded manifest. Unlike the tests run through a mach frontend jstests.py doesn't know where the objdir is, so it's hard to work out where to download the manifest. This patch adopts a heuristic approach; if the path to the jsshell looks like <root>/dist/bin and <root>/_tests exists, we assume it's a gecko-like objdir and use <root>/_tests/web-platform/ for the manifest; otherwise we just put it into the system tempdir. Because the manifest has to be updated on startup, this patch causes a startup time regression, but this will be considerably reduced by the work in Bug 1497898 for which this is a prerequisite. Differential Revision: https://phabricator.services.mozilla.com/D11667
js/src/tests/jstests.py
--- a/js/src/tests/jstests.py
+++ b/js/src/tests/jstests.py
@@ -8,16 +8,17 @@ The JS Shell Test Harness.
 
 See the adjacent README.txt for more details.
 """
 
 from __future__ import print_function
 
 import os
 import sys
+import tempfile
 import textwrap
 import platform
 from os.path import abspath, dirname, isfile, realpath
 from contextlib import contextmanager
 from copy import copy
 from itertools import chain
 from subprocess import list2cmdline, call
 
@@ -290,17 +291,17 @@ def parse_args():
     # Hide the progress bar if it will get in the way of other output.
     options.hide_progress = (options.format == 'automation' or
                              not ProgressBar.conservative_isatty() or
                              options.hide_progress)
 
     return (options, prefix, requested_paths, excluded_paths)
 
 
-def load_wpt_tests(requested_paths, excluded_paths, debug, wasm):
+def load_wpt_tests(xul_tester, requested_paths, excluded_paths):
     """Return a list of `RefTestCase` objects for the jsshell testharness.js
     tests filtered by the given paths and debug-ness."""
     repo_root = abspath(os.path.join(here, "..", "..", ".."))
     wp = os.path.join(repo_root, "testing", "web-platform")
     wpt = os.path.join(wp, "tests")
 
     sys_paths = [
         "python/mozterm",
@@ -308,16 +309,17 @@ def load_wpt_tests(requested_paths, excl
         "testing/mozbase/mozdevice",
         "testing/mozbase/mozfile",
         "testing/mozbase/mozinfo",
         "testing/mozbase/mozleak",
         "testing/mozbase/mozlog",
         "testing/mozbase/mozprocess",
         "testing/mozbase/mozprofile",
         "testing/mozbase/mozrunner",
+        "testing/web-platform/",
         "testing/web-platform/tests/tools",
         "testing/web-platform/tests/tools/third_party/html5lib",
         "testing/web-platform/tests/tools/third_party/webencodings",
         "testing/web-platform/tests/tools/wptrunner",
         "testing/web-platform/tests/tools/wptserve",
     ]
     abs_sys_paths = [os.path.join(repo_root, path) for path in sys_paths]
 
@@ -326,41 +328,54 @@ def load_wpt_tests(requested_paths, excl
         if not os.path.isdir(path):
             failed = True
             print("Could not add '%s' to the path")
     if failed:
         return []
 
     sys.path[0:0] = abs_sys_paths
 
+    import manifestupdate
     from wptrunner import products, testloader, wptcommandline, wpttest, wptlogging
 
-    wptlogging.setup({}, {})
-    kwargs = {
-        "config": None,
-        "tests_root": wpt,
-        "metadata_root": os.path.join(wp, "meta"),
+    manifest_root = tempfile.gettempdir()
+    path_split = os.path.dirname(xul_tester.js_bin).split(os.path.sep)
+    if path_split[-2:] == ["dist", "bin"]:
+        maybe_root = os.path.join(*path_split[:-2])
+        if os.path.exists(os.path.join(maybe_root, "_tests")):
+            # Assume this is a gecko objdir.
+            manifest_root = maybe_root
+
+    logger = wptlogging.setup({}, {})
+
+    manifestupdate.run(repo_root, manifest_root, logger)
+
+    kwargs = vars(wptcommandline.create_parser().parse_args([]))
+    kwargs.update({
+        "config": os.path.join(manifest_root, "_tests", "web-platform", "wptrunner.local.ini"),
         "gecko_e10s": False,
         "verify": False,
-        "wasm": wasm,
-    }
+        "wasm": xul_tester.test("wasmIsSupported()"),
+    })
     wptcommandline.set_from_config(kwargs)
     test_paths = kwargs["test_paths"]
 
     def filter_jsshell_tests(it):
         for test in it:
             if test[1].get("jsshell"):
                 yield test
 
     test_manifests = testloader.ManifestLoader(test_paths, types=["testharness"],
                                                meta_filters=[filter_jsshell_tests]).load()
 
     run_info_extras = products.load_product(kwargs["config"], "firefox")[-1](**kwargs)
-    run_info = wpttest.get_run_info(kwargs["metadata_root"], "firefox",
-                                    debug=debug, extras=run_info_extras)
+    run_info = wpttest.get_run_info(kwargs["test_paths"]["/"]["metadata_path"],
+                                    "firefox",
+                                    debug=xul_tester.test("isDebugBuild"),
+                                    extras=run_info_extras)
 
     path_filter = testloader.TestFilter(test_manifests,
                                         include=requested_paths,
                                         exclude=excluded_paths)
     loader = testloader.TestLoader(test_manifests,
                                    ["testharness"],
                                    run_info,
                                    manifest_filters=[path_filter])
@@ -410,19 +425,19 @@ def load_tests(options, requested_paths,
 
     test_dir = dirname(abspath(__file__))
     path_options = PathOptions(test_dir, requested_paths, excluded_paths)
     test_count = manifest.count_tests(test_dir, path_options)
     test_gen = manifest.load_reftests(test_dir, path_options, xul_tester)
 
     # WPT tests are already run in the browser in their own harness.
     if not options.make_manifests:
-        wpt_tests = load_wpt_tests(requested_paths, excluded_paths,
-                                   debug=xul_tester.test("isDebugBuild"),
-                                   wasm=xul_tester.test("wasmIsSupported()"))
+        wpt_tests = load_wpt_tests(xul_tester,
+                                   requested_paths,
+                                   excluded_paths)
         test_count += len(wpt_tests)
         test_gen = chain(test_gen, wpt_tests)
 
     if options.test_reflect_stringify is not None:
         def trs_gen(tests):
             for test in tests:
                 test.test_reflect_stringify = options.test_reflect_stringify
                 # Even if the test is not normally expected to pass, we still