Bug 1497898 - Update gecko wpt manifest update to use caches, r=ato
authorJames Graham <james@hoppipolla.co.uk>
Fri, 16 Nov 2018 18:48:50 +0000
changeset 503258 1a0c1f962ae46cb9669622d0ed5321398bb0de99
parent 503257 750b40bcdad7d60d10ea78f386493d962af89de1
child 503259 3dedddeb0c8d6847d9c7cb7074d7d06186182663
push id10290
push userffxbld-merge
push dateMon, 03 Dec 2018 16:23:23 +0000
treeherdermozilla-beta@700bed2445e6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersato
bugs1497898
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1497898 - Update gecko wpt manifest update to use caches, r=ato Depends on D8231 Differential Revision: https://phabricator.services.mozilla.com/D8232
testing/web-platform/manifestupdate.py
--- a/testing/web-platform/manifestupdate.py
+++ b/testing/web-platform/manifestupdate.py
@@ -1,14 +1,21 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
 import ConfigParser
 import argparse
+import hashlib
 import imp
 import os
 import sys
 
+from mozboot.util import get_state_dir
+
 from mozlog.structured import commandline
 from wptrunner.wptcommandline import set_from_config
 
 import manifestdownload
 from wptrunner import wptcommandline
 
 manifest = None
 
@@ -36,16 +43,19 @@ def create_parser():
         "--no-update", action="store_false", dest="update",
         default=True, help="Just download the manifest, don't update")
     p.add_argument(
         "--config", action="store", dest="config_path", default=None,
         help="Path to wptrunner config file")
     p.add_argument(
         "--rewrite-config", action="store_true", default=False,
         help="Force the local configuration to be regenerated")
+    p.add_argument(
+        "--cache-root", action="store", default=os.path.join(get_state_dir()[0], "cache", "wpt"),
+        help="Path to use for the metadata cache")
     commandline.add_logging_group(p)
 
     return p
 
 
 def ensure_kwargs(kwargs):
     _kwargs = vars(create_parser().parse_args([]))
     _kwargs.update(kwargs)
@@ -100,17 +110,18 @@ def run(src_root, obj_root, logger=None,
         manifestdownload.download_from_taskcluster(logger,
                                                    src_root,
                                                    test_paths,
                                                    force=force_download)
     else:
         logger.debug("Skipping manifest download")
 
     if kwargs["update"] or kwargs["rebuild"]:
-        manifests = update(logger, src_wpt_dir, test_paths, rebuild=kwargs["rebuild"])
+        manifests = update(logger, src_wpt_dir, test_paths, rebuild=kwargs["rebuild"],
+                           cache_root=kwargs["cache_root"])
     else:
         logger.debug("Skipping manifest update")
         manifests = load_manifests(test_paths)
 
     return manifests
 
 
 def ensure_manifest_directories(logger, test_paths):
@@ -159,34 +170,27 @@ def generate_config(logger, repo_root, w
     parser.set('paths', 'prefs', os.path.abspath(os.path.join(wpt_dir, parser.get("paths", "prefs"))))
 
     with open(dest_config_path, 'wb') as config_file:
         parser.write(config_file)
 
     return dest_config_path
 
 
-def update(logger, wpt_dir, test_paths, rebuild=False, config_dir=None):
+def update(logger, wpt_dir, test_paths, rebuild=False, config_dir=None, cache_root=None):
     rv = {}
-
+    wptdir_hash = hashlib.sha256(os.path.abspath(wpt_dir)).hexdigest()
     for url_base, paths in test_paths.iteritems():
-        m = None
         manifest_path = paths["manifest_path"]
-        if not rebuild and os.path.exists(manifest_path):
-            logger.info("Updating manifest %s" % manifest_path)
-            try:
-                m = manifest.manifest.load(paths["tests_path"], manifest_path)
-            except manifest.manifest.ManifestVersionMismatch:
-                logger.info("Manifest format changed, rebuilding")
-        if m is None:
-            logger.info("Recreating manifest %s" % manifest_path)
-            m = manifest.manifest.Manifest(url_base)
-        manifest.update.update(paths["tests_path"], m, working_copy=True)
-        manifest.manifest.write(m, manifest_path)
-
+        this_cache_root = os.path.join(cache_root, wptdir_hash, os.path.dirname(paths["manifest_rel_path"]))
+        m = manifest.manifest.load_and_update(paths["tests_path"],
+                                              manifest_path,
+                                              url_base,
+                                              working_copy=True,
+                                              cache_root=this_cache_root)
         path_data = {"url_base": url_base}
         path_data.update(paths)
         rv[m] = path_data
 
     return rv
 
 
 def load_manifests(test_paths):