Bug 1642446: Add an attribute to annotate toolchain tasks used by `mach bootstrap`; r=firefox-build-system-reviewers,rstewart
authorTom Prince <mozilla@hocat.ca>
Wed, 03 Jun 2020 15:55:26 +0000
changeset 597824 302d7d051a6b68b93e3b317d4244cf103dc0237d
parent 597823 ce2ec4204fe8517bf7725d162ba55b6d3297966c
child 597825 28a7557a508a85050aa6ceb7e01ded5e692df600
push id13310
push userffxbld-merge
push dateMon, 29 Jun 2020 14:50:06 +0000
treeherdermozilla-beta@15a59a0afa5c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfirefox-build-system-reviewers, rstewart
bugs1642446
milestone79.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1642446: Add an attribute to annotate toolchain tasks used by `mach bootstrap`; r=firefox-build-system-reviewers,rstewart Toolchains that are used for local development need to be built on a level-3 branch to installable via `mach bootstrap`. Add an attribute to track the fact that a toolchain is used that way, and: - ensure that everything installed via `mach boostrap` has that attribute set - ensure that everything with that attribute set is built on trunk projects We could additionally verify that attribute is only set on things used by bootstrap, but bootstrap doesn't currently have an exhaustive list of things that it might install, making that difficult. Differential Revision: https://phabricator.services.mozilla.com/D77706
python/mozboot/mozboot/base.py
python/mozbuild/mozbuild/artifact_commands.py
taskcluster/ci/toolchain/cbindgen.yml
taskcluster/ci/toolchain/clang-tidy.yml
taskcluster/ci/toolchain/clang.yml
taskcluster/ci/toolchain/dist-toolchains.yml
taskcluster/ci/toolchain/dump-syms.yml
taskcluster/ci/toolchain/fix-stacks.yml
taskcluster/ci/toolchain/minidump_stackwalk.yml
taskcluster/ci/toolchain/misc.yml
taskcluster/ci/toolchain/nasm.yml
taskcluster/ci/toolchain/node.yml
taskcluster/ci/toolchain/sccache.yml
taskcluster/docs/attributes.rst
taskcluster/taskgraph/util/verify.py
--- a/python/mozboot/mozboot/base.py
+++ b/python/mozboot/mozboot/base.py
@@ -345,17 +345,17 @@ class BaseBootstrapper(object):
 
         # If Python can't figure out what its own executable is, there's little
         # chance we're going to be able to execute mach on its own, particularly
         # on Windows.
         if not sys.executable:
             raise ValueError("cannot determine path to Python executable")
 
         cmd = [sys.executable, mach_binary, 'artifact', 'toolchain',
-               '--from-build', toolchain_job]
+               '--bootstrap', '--from-build', toolchain_job]
 
         if no_unpack:
             cmd += ['--no-unpack']
 
         subprocess.check_call(cmd, cwd=state_dir)
 
     def which(self, name, *extra_search_dirs):
         """Python implementation of which.
--- a/python/mozbuild/mozbuild/artifact_commands.py
+++ b/python/mozbuild/mozbuild/artifact_commands.py
@@ -179,25 +179,32 @@ class PackageFrontend(MachCommandBase):
     @CommandArgument('--tooltool-manifest', metavar='MANIFEST',
                      help='Explicit tooltool manifest to process')
     @CommandArgument('--authentication-file', metavar='FILE',
                      help='Use the RelengAPI token found in the given file to authenticate')
     @CommandArgument('--no-unpack', action='store_true',
                      help='Do not unpack any downloaded file')
     @CommandArgument('--retry', type=int, default=4,
                      help='Number of times to retry failed downloads')
+    @CommandArgument(
+        "--bootstrap",
+        action="store_true",
+        help="Whether this is being called from bootstrap. "
+        "This verifies the toolchain is annotated as a toolchain used for local development."
+    )
     @CommandArgument('--artifact-manifest', metavar='FILE',
                      help='Store a manifest about the downloaded taskcluster artifacts')
     @CommandArgument('files', nargs='*',
                      help='A list of files to download, in the form path@task-id, in '
                      'addition to the files listed in the tooltool manifest.')
     def artifact_toolchain(self, verbose=False, cache_dir=None,
                            skip_cache=False, from_build=(),
                            tooltool_manifest=None, authentication_file=None,
                            no_unpack=False, retry=0,
+                           bootstrap=False,
                            artifact_manifest=None, files=()):
         '''Download, cache and install pre-built toolchains.
         '''
         from mozbuild.artifacts import ArtifactCache
         from mozbuild.action.tooltool import (
             FileRecord,
             open_manifest,
             unpack_file,
@@ -321,16 +328,25 @@ class PackageFrontend(MachCommandBase):
                     b = 'toolchain-{}'.format(b)
 
                 task = toolchains.get(aliases.get(b, b))
                 if not task:
                     self.log(logging.ERROR, 'artifact', {'build': user_value},
                              'Could not find a toolchain build named `{build}`')
                     return 1
 
+                # Ensure that toolchains installed by `mach bootstrap` have the
+                # `local-toolchain attribute set. Taskgraph ensures that these
+                # are built on trunk projects, so the task will be available to
+                # install here.
+                if bootstrap and not task.attributes.get('local-toolchain'):
+                    self.log(logging.ERROR, 'artifact', {'build': user_value},
+                             'Toolchain `{build}` is not annotated as used for local development.')
+                    return 1
+
                 artifact_name = task.attributes.get('toolchain-artifact')
                 self.log(logging.DEBUG, 'artifact',
                          {'name': artifact_name,
                           'index': task.optimization.get('index-search')},
                          'Searching for {name} in {index}')
                 task_id = IndexSearch().should_replace_task(
                     task, {}, task.optimization.get('index-search', []))
                 if task_id in (True, False) or not artifact_name:
--- a/taskcluster/ci/toolchain/cbindgen.yml
+++ b/taskcluster/ci/toolchain/cbindgen.yml
@@ -1,20 +1,23 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 job-defaults:
     description: "cbindgen toolchain build"
+    attributes:
+        local-toolchain: true
     worker-type: b-linux
     worker:
         max-run-time: 3600
     run:
         script: build-cbindgen.sh
         toolchain-artifact: public/build/cbindgen.tar.xz
+    run-on-projects: [trunk]
     fetches:
         fetch:
             # If you update this, make sure to update the minimum version in
             # build/moz.configure/bindgen.configure as well.
             - cbindgen-0.14.1
 
 linux64-cbindgen:
     treeherder:
@@ -28,18 +31,16 @@ linux64-cbindgen:
             - linux64-rust-1.41
 
 macosx64-cbindgen:
     treeherder:
         symbol: TM(cbindgen)
     worker:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest"
-    run-on-projects:
-        - trunk
     run:
         arguments: ['x86_64-apple-darwin']
         resources:
             - taskcluster/scripts/misc/tooltool-download.sh
         tooltool-downloads: internal
     fetches:
         toolchain:
             - linux64-cctools-port
--- a/taskcluster/ci/toolchain/clang-tidy.yml
+++ b/taskcluster/ci/toolchain/clang-tidy.yml
@@ -1,14 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 job-defaults:
     description: "Clang-tidy build"
+    attributes:
+        local-toolchain: true
     index:
         product: static-analysis
     worker-type: b-linux-large
     worker:
         max-run-time: 3600
     run:
         toolchain-artifact: public/build/clang-tidy.tar.zst
         resources:
--- a/taskcluster/ci/toolchain/clang.yml
+++ b/taskcluster/ci/toolchain/clang.yml
@@ -44,18 +44,21 @@ linux64-clang-7:
         fetch:
             - clang-7
         toolchain:
             - linux64-binutils
             - linux64-gcc-7
 
 linux64-clang-9:
     description: "Clang 9 toolchain build"
+    attributes:
+        local-toolchain: true
     treeherder:
         symbol: TL(clang9)
+    run-on-projects: [trunk]
     run:
         using: toolchain-script
         script: build-clang.sh
         arguments:
             - 'build/build-clang/clang-linux64.json'
         resources:
             - 'build/build-clang/clang-linux64.json'
         toolchain-alias: linux64-clang
@@ -230,16 +233,18 @@ linux64-clang-10:
             - clang-10
         toolchain:
             - linux64-binutils
             - linux64-gcc-7
             - wasi-sysroot
 
 macosx64-clang:
     description: "Clang toolchain build"
+    attributes:
+        local-toolchain: true
     treeherder:
         symbol: TM(clang)
     worker-type: b-linux-large
     worker:
         max-run-time: 3600
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest"
     run-on-projects:
@@ -260,22 +265,26 @@ macosx64-clang:
             - linux64-binutils
             - linux64-cctools-port
             - linux64-clang-9
             - linux64-gcc-7
             - linux64-node
 
 win64-clang-cl:
     description: "Clang-cl toolchain build"
+    attributes:
+        local-toolchain: true
     treeherder:
         symbol: TW64(clang-cl)
     worker-type: b-win2012
     worker:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
+    run-on-projects:
+        - trunk
     run:
         script: build-clang.sh
         arguments:
             - 'build/build-clang/clang-win64.json'
         resources:
             - 'build/build-clang/clang-win64.json'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/clang.tar.zst
--- a/taskcluster/ci/toolchain/dist-toolchains.yml
+++ b/taskcluster/ci/toolchain/dist-toolchains.yml
@@ -1,14 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 job-defaults:
     description: "sccache-dist toolchain archive build"
+    attributes:
+        local-toolchain: true
     worker-type: b-linux
     run-on-projects:
         - trunk
     run:
         using: toolchain-script
         script: build-dist-toolchains.sh
 
 clang-dist-toolchain:
--- a/taskcluster/ci/toolchain/dump-syms.yml
+++ b/taskcluster/ci/toolchain/dump-syms.yml
@@ -1,16 +1,17 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 job-defaults:
     description: "dump_syms toolchain build"
     worker:
         max-run-time: 1800
+    run-on-projects: [trunk]
     run:
         script: build-dump-syms.sh
     fetches:
         fetch:
             - dump-syms
 
 linux64-dump-syms:
     treeherder:
@@ -20,16 +21,18 @@ linux64-dump-syms:
         toolchain-artifact: public/build/dump_syms.tar.xz
     fetches:
         toolchain:
             - linux64-binutils
             - linux64-clang
             - linux64-rust
 
 win64-dump-syms:
+    attributes:
+        local-toolchain: true
     treeherder:
         symbol: TW64(dump_syms)
     worker-type: b-win2012
     worker:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
     run:
         toolchain-artifact: public/build/dump_syms.tar.bz2
--- a/taskcluster/ci/toolchain/fix-stacks.yml
+++ b/taskcluster/ci/toolchain/fix-stacks.yml
@@ -1,14 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 job-defaults:
     description: "fix-stacks toolchain build"
+    attributes:
+        local-toolchain: true
     worker-type: b-linux
     worker:
         max-run-time: 1800
     run:
         script: build-fix-stacks.sh
     run-on-projects:
         - trunk
     fetches:
--- a/taskcluster/ci/toolchain/minidump_stackwalk.yml
+++ b/taskcluster/ci/toolchain/minidump_stackwalk.yml
@@ -1,14 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 job-defaults:
     description: "minidump_stackwalk toolchain build"
+    attributes:
+        local-toolchain: true
     worker-type: b-linux
     worker:
         max-run-time: 1800
     run:
         script: build-minidump-stackwalk.sh
         sparse-profile: null
         resources:
             - 'build/moz.configure'
--- a/taskcluster/ci/toolchain/misc.yml
+++ b/taskcluster/ci/toolchain/misc.yml
@@ -137,18 +137,21 @@ browsertime:
             - 'tools/browsertime/mach_commands.py'
         toolchain-artifact: public/build/browsertime.tar.bz2
     fetches:
         toolchain:
             - linux64-node
 
 wasi-sysroot:
     description: "wasi sysroot build process"
+    attributes:
+        local-toolchain: true
     treeherder:
         symbol: TL(wasi-sysroot)
+    run-on-projects: [trunk]
     run:
         script: build-wasi-sysroot.sh
         sparse-profile: null
         toolchain-artifact: public/build/wasi-sysroot.tar.xz
     fetches:
         fetch:
             - clang-9
             - wasi-sdk
@@ -189,18 +192,21 @@ wgpu-deps:
     fetches:
         fetch:
             - android-rs-glue
         toolchain:
             - linux64-rust-1.41  # whatever m-c is built with
 
 linux64-lucetc:
     description: "lucetc build process"
+    attributes:
+        local-toolchain: true
     treeherder:
         symbol: TL(lucetc)
+    run-on-projects: [trunk]
     run:
         script: build-lucetc.sh
         toolchain-artifact: public/build/lucetc.tar.xz
     fetches:
         fetch:
             - cmake
             - lucetc-source
         toolchain:
--- a/taskcluster/ci/toolchain/nasm.yml
+++ b/taskcluster/ci/toolchain/nasm.yml
@@ -1,36 +1,41 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 job-defaults:
     worker-type: b-linux
     worker:
         max-run-time: 1800
+    run-on-projects: [trunk]
     run:
         toolchain-artifact: public/build/nasm.tar.bz2
 
 win64-nasm:
     description: "nasm win64 build"
+    attributes:
+        local-toolchain: true
     treeherder:
         symbol: TW64(nasm)
     worker:
         docker-image: {in-tree: mingw32-build}
     run:
         script: build-nasm.sh
         arguments: ['win64']
     fetches:
         fetch:
             - nasm-2.14.02
         toolchain:
             - linux64-clang-mingw-x64
 
 linux64-nasm:
     description: "nasm linux64 build"
+    attributes:
+        local-toolchain: true
     treeherder:
         symbol: TL(nasm)
     run:
         script: build-nasm.sh
         arguments: ['linux64']
     fetches:
         fetch:
             - nasm-2.14.02
--- a/taskcluster/ci/toolchain/node.yml
+++ b/taskcluster/ci/toolchain/node.yml
@@ -1,33 +1,34 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 job-defaults:
     description: "Node repack toolchain build"
+    attributes:
+        local-toolchain: true
     worker-type: b-linux
     worker:
         max-run-time: 1800
+    run-on-projects: [trunk]
     run:
         script: repack-node.sh
         toolchain-artifact: public/build/node.tar.xz
 
 linux64-node-10:
     treeherder:
         symbol: TL(node-10)
     run:
         arguments: ['linux64', '10']
         toolchain-alias: linux64-node
 
 macosx64-node-10:
     treeherder:
         symbol: TM(node-10)
-    run-on-projects:
-        - trunk
     run:
         arguments: ['macosx64', '10']
         toolchain-alias: macosx64-node
 
 win64-node-10:
     treeherder:
         symbol: TW64(node-10)
     run:
@@ -37,10 +38,8 @@ win64-node-10:
 
 win32-node-10:
     treeherder:
         symbol: TW32(node-10)
     run:
         arguments: ['win32', '10']
         toolchain-artifact: public/build/node.tar.bz2
         toolchain-alias: win32-node
-    run-on-projects:
-        - trunk
--- a/taskcluster/ci/toolchain/sccache.yml
+++ b/taskcluster/ci/toolchain/sccache.yml
@@ -1,16 +1,19 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 job-defaults:
     description: "sccache toolchain build"
+    attributes:
+        local-toolchain: true
     treeherder:
         symbol: TL(sccache)
+    run-on-projects: [trunk]
     run:
         using: toolchain-script
         script: build-sccache.sh
     fetches:
         fetch:
             - sccache
 
 linux64-sccache:
@@ -29,18 +32,16 @@ linux64-sccache:
 macosx64-sccache:
     treeherder:
         symbol: TM(sccache)
     worker-type: b-linux
     worker:
         max-run-time: 1800
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest"
-    run-on-projects:
-        - trunk
     run:
         arguments: ['x86_64-apple-darwin']
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         tooltool-downloads: internal
         toolchain-artifact: public/build/sccache.tar.xz
     fetches:
         toolchain:
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -390,8 +390,13 @@ geckodriver
 If non-empty, declares that the (toolchain) task is a `geckodriver`
 task that produces a binary that should be signed.
 
 rebuild-on-release
 ==================
 If true, the digest for this task will also depend on if the branch is a
 release branch.  This will cause tasks like toolchains to be rebuilt as they
 move from e.g. autoland to mozilla-central.
+
+local-toolchain
+===============
+This toolchain is used for local development, so should be built on trunk, even
+if it does not have any in-graph consumers.
--- a/taskcluster/taskgraph/util/verify.py
+++ b/taskcluster/taskgraph/util/verify.py
@@ -333,8 +333,29 @@ def verify_test_packaging(task, taskgrap
                             'Unset MOZ_AUTOMATION_PACKAGE_TESTS in the task definition '
                             'to fix.'.format(task.label, package_tests))
         if exceptions:
             raise Exception("\n".join(exceptions))
         return
     if task.kind == 'test':
         build_task = taskgraph[task.dependencies['build']]
         scratch_pad[build_task.label] = 1
+
+
+@verifications.add('full_task_graph')
+def verify_local_toolchains(task, taskgraph, scratch_pad, graph_config):
+    """
+    Toolchains that are used for local development need to be built on a
+    level-3 branch to installable via `mach bootstrap`. We ensure here that all
+    such tasks run on at least trunk projects, even if they aren't pulled in as
+    a dependency of other tasks in the graph.
+
+    There is code in `mach artifact toolchain` that verifies that anything
+    installed via `mach bootstrap` has the attribute set.
+    """
+    if task and task.attributes.get('local-toolchain'):
+        run_on_projects = task.attributes.get('run_on_projects', [])
+        if not any(alias in run_on_projects for alias in ["all", "trunk"]):
+            raise Exception(
+                "Toolchain {} used for local development is not built on trunk. {}".format(
+                    task.label, run_on_projects
+                )
+            )