Bug 1540540 - Do not unconditionally re-extract stuff from mach bootstrap. r=glandium
authorEmilio Cobos Álvarez <emilio@crisal.io>
Wed, 03 Apr 2019 07:08:12 +0000
changeset 467716 a566275dc806386e01704260d49beda426b5ed5c
parent 467715 58939777811ac6f9b7e7ca155aaf92c6f0478a21
child 467717 1ec7f80cf258370b47afbe6de253bad316ad80c2
push id35810
push useraciure@mozilla.com
push dateThu, 04 Apr 2019 04:33:36 +0000
treeherdermozilla-central@b72c02e34261 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1540540
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1540540 - Do not unconditionally re-extract stuff from mach bootstrap. r=glandium Keep a checksum to avoid re-extracting artifacts over and over. In the future we can also check the checksum to avoid downloading the artifact altogether, maybe. Differential Revision: https://phabricator.services.mozilla.com/D25542
python/mozboot/mozboot/base.py
python/mozbuild/mozbuild/action/tooltool.py
--- a/python/mozboot/mozboot/base.py
+++ b/python/mozboot/mozboot/base.py
@@ -284,22 +284,18 @@ class BaseBootstrapper(object):
         '''
         Install any necessary packages needed to supply NodeJS'''
         raise NotImplementedError(
             '%s does not yet implement ensure_node_packages()'
             % __name__)
 
     def install_toolchain_static_analysis(self, state_dir, checkout_root, toolchain_job):
         clang_tools_path = os.path.join(state_dir, 'clang-tools')
-        import shutil
-        if os.path.exists(clang_tools_path):
-            shutil.rmtree(clang_tools_path)
-
-        # Re-create the directory for clang_tools
-        os.mkdir(clang_tools_path)
+        if not os.path.exists(clang_tools_path):
+            os.mkdir(clang_tools_path)
         self.install_toolchain_artifact(clang_tools_path, checkout_root, toolchain_job)
 
     def install_toolchain_artifact(self, state_dir, checkout_root, toolchain_job):
         mach_binary = os.path.join(checkout_root, 'mach')
         mach_binary = os.path.abspath(mach_binary)
         if not os.path.exists(mach_binary):
             raise ValueError("mach not found at %s" % mach_binary)
 
--- a/python/mozbuild/mozbuild/action/tooltool.py
+++ b/python/mozbuild/mozbuild/action/tooltool.py
@@ -514,46 +514,78 @@ def fetch_file(base_urls, file_record, g
 
 def clean_path(dirname):
     """Remove a subtree if is exists. Helper for unpack_file()."""
     if os.path.exists(dirname):
         log.info('rm tree: %s' % dirname)
         shutil.rmtree(dirname)
 
 
+CHECKSUM_SUFFIX = ".checksum"
+
+
+def _cache_checksum_matches(base_file, checksum):
+    try:
+        with open(base_file + CHECKSUM_SUFFIX, "rb") as f:
+            prev_checksum = f.read().strip()
+            if prev_checksum == checksum:
+                log.info("Cache matches, avoiding extracting in '%s'" % base_file)
+                return True
+            return False
+    except IOError as e:
+        return False
+
+
+def _compute_cache_checksum(filename):
+    with open(filename, "rb") as f:
+        return digest_file(f, "sha256")
+
+
 def unpack_file(filename, setup=None):
     """Untar `filename`, assuming it is uncompressed or compressed with bzip2,
     xz, gzip, or unzip a zip file. The file is assumed to contain a single
     directory with a name matching the base of the given filename.
     Xz support is handled by shelling out to 'tar'."""
+
+    checksum = _compute_cache_checksum(filename)
+
     if tarfile.is_tarfile(filename):
         tar_file, zip_ext = os.path.splitext(filename)
         base_file, tar_ext = os.path.splitext(tar_file)
+        if _cache_checksum_matches(base_file, checksum):
+            return True
         clean_path(base_file)
         log.info('untarring "%s"' % filename)
         tar = tarfile.open(filename)
         tar.extractall()
         tar.close()
     elif filename.endswith('.tar.xz'):
         base_file = filename.replace('.tar.xz', '')
+        if _cache_checksum_matches(base_file, checksum):
+            return True
         clean_path(base_file)
         log.info('untarring "%s"' % filename)
         if not execute('tar -Jxf %s 2>&1' % filename):
             return False
     elif zipfile.is_zipfile(filename):
         base_file = filename.replace('.zip', '')
+        if _cache_checksum_matches(base_file, checksum):
+            return True
         clean_path(base_file)
         log.info('unzipping "%s"' % filename)
         z = zipfile.ZipFile(filename)
         z.extractall()
         z.close()
     else:
         log.error("Unknown archive extension for filename '%s'" % filename)
         return False
 
+    with open(base_file + CHECKSUM_SUFFIX, "wb") as f:
+        f.write(checksum)
+
     if setup and not execute(os.path.join(base_file, setup)):
         return False
     return True
 
 
 def fetch_files(manifest_file, base_urls, filenames=[], cache_folder=None,
                 auth_file=None, region=None):
     # Lets load the manifest file