Bug 1497898 - Update gecko wpt manifest update to use caches, r=ato
☠☠ backed out by fe1c2bb6cfbc ☠ ☠
authorJames Graham <james@hoppipolla.co.uk>
Wed, 10 Oct 2018 17:51:42 +0000
changeset 500629 57877c614829ff051ed0e2b66f895292e904b207
parent 500628 22a06c8c8dc6ea3a7413904dc486eb2bc2ac2f92
child 500630 d46116aa35273c5da6a5970ee683fedaa81dd628
push id1864
push userffxbld-merge
push dateMon, 03 Dec 2018 15:51:40 +0000
treeherdermozilla-release@f040763d99ad [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersato
bugs1497898
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1497898 - Update gecko wpt manifest update to use caches, r=ato Depends on D8231 Differential Revision: https://phabricator.services.mozilla.com/D8232
testing/web-platform/mach_commands_base.py
testing/web-platform/manifestupdate.py
--- a/testing/web-platform/mach_commands_base.py
+++ b/testing/web-platform/mach_commands_base.py
@@ -1,14 +1,17 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import os
 import sys
 
+from mozboot.util import get_state_dir
+
 
 def create_parser_wpt():
     from wptrunner import wptcommandline
     return wptcommandline.create_parser(["fennec", "firefox", "chrome", "edge", "servo"])
 
 
 class WebPlatformTestsRunner(object):
     """Run web platform tests."""
@@ -39,9 +42,10 @@ class WebPlatformTestsRunner(object):
         result = wptrunner.start(**kwargs)
         return int(not result)
 
     def update_manifest(self, logger, **kwargs):
         import manifestupdate
         return manifestupdate.run(logger=logger,
                                   src_root=self.setup.topsrcdir,
                                   obj_root=self.setup.topobjdir,
+                                  cache_root=os.path.join(get_state_dir()[0], "cache", "wpt"),
                                   **kwargs)
--- a/testing/web-platform/manifestupdate.py
+++ b/testing/web-platform/manifestupdate.py
@@ -1,10 +1,11 @@
 import ConfigParser
 import argparse
+import hashlib
 import imp
 import os
 import sys
 
 from mozlog.structured import commandline
 from wptrunner.wptcommandline import set_from_config
 
 import manifestdownload
@@ -100,17 +101,18 @@ def run(src_root, obj_root, logger=None,
         manifestdownload.download_from_taskcluster(logger,
                                                    src_root,
                                                    test_paths,
                                                    force=force_download)
     else:
         logger.debug("Skipping manifest download")
 
     if kwargs["update"] or kwargs["rebuild"]:
-        manifests = update(logger, src_wpt_dir, test_paths, rebuild=kwargs["rebuild"])
+        manifests = update(logger, src_wpt_dir, test_paths, rebuild=kwargs["rebuild"],
+                           cache_root=kwargs["cache_root"])
     else:
         logger.debug("Skipping manifest update")
         manifests = load_manifests(test_paths)
 
     return manifests
 
 
 def ensure_manifest_directories(logger, test_paths):
@@ -159,34 +161,27 @@ def generate_config(logger, repo_root, w
     parser.set('paths', 'prefs', os.path.abspath(os.path.join(wpt_dir, parser.get("paths", "prefs"))))
 
     with open(dest_config_path, 'wb') as config_file:
         parser.write(config_file)
 
     return dest_config_path
 
 
-def update(logger, wpt_dir, test_paths, rebuild=False, config_dir=None):
+def update(logger, wpt_dir, test_paths, rebuild=False, config_dir=None, cache_root=None):
     rv = {}
-
+    wptdir_hash = hashlib.sha256(os.path.abspath(wpt_dir)).hexdigest()
     for url_base, paths in test_paths.iteritems():
-        m = None
         manifest_path = paths["manifest_path"]
-        if not rebuild and os.path.exists(manifest_path):
-            logger.info("Updating manifest %s" % manifest_path)
-            try:
-                m = manifest.manifest.load(paths["tests_path"], manifest_path)
-            except manifest.manifest.ManifestVersionMismatch:
-                logger.info("Manifest format changed, rebuilding")
-        if m is None:
-            logger.info("Recreating manifest %s" % manifest_path)
-            m = manifest.manifest.Manifest(url_base)
-        manifest.update.update(paths["tests_path"], m, working_copy=True)
-        manifest.manifest.write(m, manifest_path)
-
+        this_cache_root = os.path.join(cache_root, wptdir_hash, os.path.dirname(paths["manifest_rel_path"]))
+        m = manifest.manifest.load_and_update(paths["tests_path"],
+                                              manifest_path,
+                                              url_base,
+                                              working_copy=True,
+                                              cache_root=this_cache_root)
         path_data = {"url_base": url_base}
         path_data.update(paths)
         rv[m] = path_data
 
     return rv
 
 
 def load_manifests(test_paths):