Bug 1383996 - Add an argument to `mach artifact toolchain` to store a manifest for chain-of-trust validation. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Thu, 20 Jul 2017 17:52:56 +0900
changeset 419642 6a49ccfbabeba32d0fefa53c95bf7478d1b4a206
parent 419641 7ad0f4024a8a779622fa79db42a9423c0150696e
child 419643 ba130cb1f4270176a013f69c5f754a65ae81d108
push id7566
push usermtabara@mozilla.com
push dateWed, 02 Aug 2017 08:25:16 +0000
treeherdermozilla-beta@86913f512c3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1383996, 1383993
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1383996 - Add an argument to `mach artifact toolchain` to store a manifest for chain-of-trust validation. r=gps Chain of trust validation will require to know what the inputs for a given build are, and mach artifact toolchain fetches such inputs. So we make it output a manifest that the chain of trust validation process will be able to use and correlate with other information, such as the one from bug 1383993. At the same time, we make the produced manifest contain information about tooltool-downloaded packages, which will allow to track the progress in the migration from tooltool to TC artifacts.
python/mozbuild/mozbuild/mach_commands.py
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -2,16 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
 import collections
 import errno
+import hashlib
 import itertools
 import json
 import logging
 import operator
 import os
 import subprocess
 import sys
 
@@ -33,16 +34,17 @@ from mozbuild.base import (
     BuildEnvironmentNotFoundException,
     MachCommandBase,
     MachCommandConditions as conditions,
     MozbuildObject,
     MozconfigFindException,
     MozconfigLoadException,
     ObjdirMismatchException,
 )
+from mozbuild.util import ensureParentDir
 
 from mozbuild.backend import (
     backends,
     get_backend_class,
 )
 from mozbuild.shellutil import quote as shell_quote
 
 
@@ -1686,24 +1688,26 @@ class PackageFrontend(MachCommandBase):
     @CommandArgument('--authentication-file', metavar='FILE',
         help='Use the RelengAPI token found in the given file to authenticate')
     @CommandArgument('--tooltool-url', metavar='URL',
         help='Use the given url as tooltool server')
     @CommandArgument('--no-unpack', action='store_true',
         help='Do not unpack any downloaded file')
     @CommandArgument('--retry', type=int, default=0,
         help='Number of times to retry failed downloads')
+    @CommandArgument('--artifact-manifest', metavar='FILE',
+        help='Store a manifest about the downloaded taskcluster artifacts')
     @CommandArgument('files', nargs='*',
         help='A list of files to download, in the form path@task-id, in '
              'addition to the files listed in the tooltool manifest.')
     def artifact_toolchain(self, verbose=False, cache_dir=None,
                           skip_cache=False, from_build=(),
                           tooltool_manifest=None, authentication_file=None,
                           tooltool_url=None, no_unpack=False, retry=None,
-                          files=()):
+                          artifact_manifest=None, files=()):
         '''Download, cache and install pre-built toolchains.
         '''
         from mozbuild.artifacts import ArtifactCache
         from mozbuild.action.tooltool import (
             FileRecord,
             open_manifest,
             unpack_file,
         )
@@ -1911,37 +1915,57 @@ class PackageFrontend(MachCommandBase):
                 downloaded.append(record)
                 break
 
             if not valid:
                 self.log(logging.ERROR, 'artifact', {'name': record.basename},
                          'Failed to download {name}')
                 return 1
 
+        artifacts = {} if artifact_manifest else None
+
         for record in downloaded:
             local = os.path.join(os.getcwd(), record.basename)
             if os.path.exists(local):
                 os.unlink(local)
             # unpack_file needs the file with its final name to work
             # (https://github.com/mozilla/build-tooltool/issues/38), so we
             # need to copy it, even though we remove it later. Use hard links
             # when possible.
             try:
                 os.link(record.filename, local)
             except Exception:
                 shutil.copy(record.filename, local)
+            # Keep a sha256 of each downloaded file, for the chain-of-trust
+            # validation.
+            if artifact_manifest is not None:
+                with open(local) as fh:
+                    h = hashlib.sha256()
+                    while True:
+                        data = fh.read(1024 * 1024)
+                        if not data:
+                            break
+                        h.update(data)
+                artifacts[record.url] = {
+                    'sha256': h.hexdigest(),
+                }
             if record.unpack and not no_unpack:
                 unpack_file(local, record.setup)
                 os.unlink(local)
 
         if not downloaded:
             self.log(logging.ERROR, 'artifact', {}, 'Nothing to download')
             if files:
                 return 1
 
+        if artifacts:
+            ensureParentDir(artifact_manifest)
+            with open(artifact_manifest, 'w') as fh:
+                json.dump(artifacts, fh, indent=4, sort_keys=True)
+
         return 0
 
 
 @CommandProvider
 class Vendor(MachCommandBase):
     """Vendor third-party dependencies into the source repository."""
 
     @Command('vendor', category='misc',