Bug 1497898 - Update gecko wpt manifest update to use caches, r=ato
authorJames Graham <james@hoppipolla.co.uk>
Fri, 16 Nov 2018 18:48:50 +0000
changeset 446822 1a0c1f962ae46cb9669622d0ed5321398bb0de99
parent 446821 750b40bcdad7d60d10ea78f386493d962af89de1
child 446823 3dedddeb0c8d6847d9c7cb7074d7d06186182663
push id35052
push userapavel@mozilla.com
push dateSat, 17 Nov 2018 11:25:40 +0000
treeherdermozilla-central@efc1da42132b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1497898 - Update gecko wpt manifest update to use caches, r=ato Depends on D8231 Differential Revision: https://phabricator.services.mozilla.com/D8232
--- a/testing/web-platform/manifestupdate.py
+++ b/testing/web-platform/manifestupdate.py
@@ -1,14 +1,21 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import ConfigParser
 import argparse
+import hashlib
 import imp
 import os
 import sys
+from mozboot.util import get_state_dir
 from mozlog.structured import commandline
 from wptrunner.wptcommandline import set_from_config
 import manifestdownload
 from wptrunner import wptcommandline
 manifest = None
@@ -36,16 +43,19 @@ def create_parser():
         "--no-update", action="store_false", dest="update",
         default=True, help="Just download the manifest, don't update")
         "--config", action="store", dest="config_path", default=None,
         help="Path to wptrunner config file")
         "--rewrite-config", action="store_true", default=False,
         help="Force the local configuration to be regenerated")
+    p.add_argument(
+        "--cache-root", action="store", default=os.path.join(get_state_dir()[0], "cache", "wpt"),
+        help="Path to use for the metadata cache")
     return p
 def ensure_kwargs(kwargs):
     _kwargs = vars(create_parser().parse_args([]))
@@ -100,17 +110,18 @@ def run(src_root, obj_root, logger=None,
         logger.debug("Skipping manifest download")
     if kwargs["update"] or kwargs["rebuild"]:
-        manifests = update(logger, src_wpt_dir, test_paths, rebuild=kwargs["rebuild"])
+        manifests = update(logger, src_wpt_dir, test_paths, rebuild=kwargs["rebuild"],
+                           cache_root=kwargs["cache_root"])
         logger.debug("Skipping manifest update")
         manifests = load_manifests(test_paths)
     return manifests
 def ensure_manifest_directories(logger, test_paths):
@@ -159,34 +170,27 @@ def generate_config(logger, repo_root, w
     parser.set('paths', 'prefs', os.path.abspath(os.path.join(wpt_dir, parser.get("paths", "prefs"))))
     with open(dest_config_path, 'wb') as config_file:
     return dest_config_path
-def update(logger, wpt_dir, test_paths, rebuild=False, config_dir=None):
+def update(logger, wpt_dir, test_paths, rebuild=False, config_dir=None, cache_root=None):
     rv = {}
+    wptdir_hash = hashlib.sha256(os.path.abspath(wpt_dir)).hexdigest()
     for url_base, paths in test_paths.iteritems():
-        m = None
         manifest_path = paths["manifest_path"]
-        if not rebuild and os.path.exists(manifest_path):
-            logger.info("Updating manifest %s" % manifest_path)
-            try:
-                m = manifest.manifest.load(paths["tests_path"], manifest_path)
-            except manifest.manifest.ManifestVersionMismatch:
-                logger.info("Manifest format changed, rebuilding")
-        if m is None:
-            logger.info("Recreating manifest %s" % manifest_path)
-            m = manifest.manifest.Manifest(url_base)
-        manifest.update.update(paths["tests_path"], m, working_copy=True)
-        manifest.manifest.write(m, manifest_path)
+        this_cache_root = os.path.join(cache_root, wptdir_hash, os.path.dirname(paths["manifest_rel_path"]))
+        m = manifest.manifest.load_and_update(paths["tests_path"],
+                                              manifest_path,
+                                              url_base,
+                                              working_copy=True,
+                                              cache_root=this_cache_root)
         path_data = {"url_base": url_base}
         rv[m] = path_data
     return rv
 def load_manifests(test_paths):