Bug 1382564 - Use the information from bug 1374940 with `mach artifact toolchain`. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Wed, 21 Jun 2017 15:18:33 +0900
changeset 419009 5e1c7ab7ec0768714acb3c3f79b9ced64e5b639c
parent 419008 b19a95c5c5b758ac90d50be3bd137dd3b2c64d75
child 419010 5b8225dac5532f0726c9a5f9bf81e97ce8367331
push id7566
push usermtabara@mozilla.com
push dateWed, 02 Aug 2017 08:25:16 +0000
treeherdermozilla-beta@86913f512c3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1382564, 1374940
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1382564 - Use the information from bug 1374940 with `mach artifact toolchain`. r=gps Bug 1374940 adds a MOZ_TOOLCHAINS environment variable with a list of path@task-id strings, where task-id is corresponding to the (possibly optimized) toolchain job, and path corresponding to the toolchain-artifact defined for that toolchain job. We want to use that to pull artifacts instead of tooltool packages.
js/src/devtools/automation/winbuildenv.sh
python/mozbuild/mozbuild/mach_commands.py
taskcluster/scripts/builder/build-haz-linux.sh
taskcluster/scripts/builder/sm-tooltool-config.sh
taskcluster/scripts/misc/build-clang-windows-helper32.sh
taskcluster/scripts/misc/build-clang-windows-helper64.sh
taskcluster/scripts/misc/tooltool-download.sh
testing/mozharness/mozharness/mozilla/building/buildbase.py
testing/mozharness/mozharness/mozilla/tooltool.py
testing/mozharness/scripts/desktop_l10n.py
--- a/js/src/devtools/automation/winbuildenv.sh
+++ b/js/src/devtools/automation/winbuildenv.sh
@@ -26,11 +26,11 @@ else
 fi
 
 # PATH also needs to point to mozmake.exe, which can come from either
 # newer mozilla-build or tooltool.
 if ! which mozmake 2>/dev/null; then
     export PATH="$PATH:$SOURCE/.."
     if ! which mozmake 2>/dev/null; then
   TT_SERVER=${TT_SERVER:-https://api.pub.build.mozilla.org/tooltool/}
-  ( cd $SOURCE/..; $SOURCE/mach artifact toolchain -v --tooltool-manifest $SOURCE/browser/config/tooltool-manifests/${platform:-win32}/releng.manifest --tooltool-url $TT_SERVER --retry 4${TOOLTOOL_CACHE:+ --cache-dir ${TOOLTOOL_CACHE}})
+  ( cd $SOURCE/..; $SOURCE/mach artifact toolchain -v --tooltool-manifest $SOURCE/browser/config/tooltool-manifests/${platform:-win32}/releng.manifest --tooltool-url $TT_SERVER --retry 4${TOOLTOOL_CACHE:+ --cache-dir ${TOOLTOOL_CACHE}}${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}})
     fi
 fi
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1687,17 +1687,18 @@ class PackageFrontend(MachCommandBase):
         help='Use the RelengAPI token found in the given file to authenticate')
     @CommandArgument('--tooltool-url', metavar='URL',
         help='Use the given url as tooltool server')
     @CommandArgument('--no-unpack', action='store_true',
         help='Do not unpack any downloaded file')
     @CommandArgument('--retry', type=int, default=0,
         help='Number of times to retry failed downloads')
     @CommandArgument('files', nargs='*',
-        help='Only download the given file names (you may use file name stems)')
+        help='A list of files to download, in the form path@task-id, in '
+             'addition to the files listed in the tooltool manifest.')
     def artifact_toolchain(self, verbose=False, cache_dir=None,
                           skip_cache=False, from_build=(),
                           tooltool_manifest=None, authentication_file=None,
                           tooltool_url=None, no_unpack=False, retry=None,
                           files=()):
         '''Download, cache and install pre-built toolchains.
         '''
         from mozbuild.artifacts import ArtifactCache
@@ -1837,16 +1838,31 @@ class PackageFrontend(MachCommandBase):
                     name = name[len('public/'):]
                     if name.startswith('logs/'):
                         continue
                     name = os.path.basename(name)
                     records[name] = DownloadRecord(
                         get_artifact_url(task_id, artifact['name']),
                         name, None, None, None, unpack=True)
 
+        # Handle the list of files of the form path@task-id on the command
+        # line. Each of those give a path to an artifact to download.
+        # For backwards compatibility with mozboot's
+        # install_tooltool_clang_package (until that is migrated to use
+        # --from-build), files without a @ are considered as a filter over the
+        # tooltool manifest contents.
+        for f in files:
+            if '@' in f:
+                name, task_id = f.rsplit('@', 1)
+                records[name] = DownloadRecord(
+                    get_artifact_url(task_id, name), os.path.basename(name),
+                    None, None, None, unpack=True)
+
+        files = tuple(f for f in files if '@' not in f)
+
         for record in records.itervalues():
             if files and not any(record.basename == f or
                                       record.basename.startswith('%s.' % f)
                                       for f in files):
                 continue
 
             self.log(logging.INFO, 'artifact', {'name': record.basename},
                      'Downloading {name}')
--- a/taskcluster/scripts/builder/build-haz-linux.sh
+++ b/taskcluster/scripts/builder/build-haz-linux.sh
@@ -48,17 +48,17 @@ GECKO_DIR=$( cd "$GECKO_DIR" && pwd )
 # Directory to populate with tooltool-installed tools
 export TOOLTOOL_DIR="$WORKSPACE"
 
 # Directory to hold the (useless) object files generated by the analysis.
 export MOZ_OBJDIR="$WORKSPACE/obj-analyzed"
 mkdir -p "$MOZ_OBJDIR"
 
 if [ -n "$DO_TOOLTOOL" ]; then
-  ( cd $TOOLTOOL_DIR; $GECKO_DIR/mach artifact toolchain -v --tooltool-url https://api.pub.build.mozilla.org/tooltool/ --tooltool-manifest $GECKO_DIR/$TOOLTOOL_MANIFEST --cache-dir $TOOLTOOL_CACHE )
+  ( cd $TOOLTOOL_DIR; $GECKO_DIR/mach artifact toolchain -v --tooltool-url https://api.pub.build.mozilla.org/tooltool/ --tooltool-manifest $GECKO_DIR/$TOOLTOOL_MANIFEST --cache-dir $TOOLTOOL_CACHE${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}} )
 fi
 
 export NO_MERCURIAL_SETUP_CHECK=1
 
 if [[ "$PROJECT" = "browser" ]]; then (
     cd "$WORKSPACE"
     set "$WORKSPACE"
     . setup-ccache.sh
--- a/taskcluster/scripts/builder/sm-tooltool-config.sh
+++ b/taskcluster/scripts/builder/sm-tooltool-config.sh
@@ -41,9 +41,9 @@ esac
 # Install everything needed for the browser on this platform. Not all of it is
 # necessary for the JS shell, but it's less duplication to share tooltool
 # manifests.
 BROWSER_PLATFORM=$PLATFORM_OS$BITS
 
 : ${TOOLTOOL_CHECKOUT:=$WORK}
 export TOOLTOOL_CHECKOUT
 
-(cd $TOOLTOOL_CHECKOUT && ${SRCDIR}/mach artifact toolchain -v --tooltool-url $TOOLTOOL_SERVER --tooltool-manifest $SRCDIR/$TOOLTOOL_MANIFEST ${TOOLTOOL_CACHE:+ --cache-dir $TOOLTOOL_CACHE})
+(cd $TOOLTOOL_CHECKOUT && ${SRCDIR}/mach artifact toolchain -v --tooltool-url $TOOLTOOL_SERVER --tooltool-manifest $SRCDIR/$TOOLTOOL_MANIFEST ${TOOLTOOL_CACHE:+ --cache-dir $TOOLTOOL_CACHE}${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}})
--- a/taskcluster/scripts/misc/build-clang-windows-helper32.sh
+++ b/taskcluster/scripts/misc/build-clang-windows-helper32.sh
@@ -8,17 +8,17 @@ set -x -e -v
 export TOOLTOOL_CACHE
 
 TOOLTOOL_AUTH_FILE=/c/builds/relengapi.tok
 if [ ! -e ${TOOLTOOL_AUTH_FILE} ]; then
     echo cannot find ${TOOLTOOL_AUTH_FILE}
     exit 1
 fi
 
-./build/src/mach artifact toolchain -v --authentication-file="${TOOLTOOL_AUTH_FILE}" --tooltool-manifest "build/src/${TOOLTOOL_MANIFEST}"${TOOLTOOL_CACHE:+ --cache-dir ${TOOLTOOL_CACHE}}
+./build/src/mach artifact toolchain -v --authentication-file="${TOOLTOOL_AUTH_FILE}" --tooltool-manifest "build/src/${TOOLTOOL_MANIFEST}"${TOOLTOOL_CACHE:+ --cache-dir ${TOOLTOOL_CACHE}}${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}}
 
 # Set up all the Visual Studio paths.
 MSVC_DIR=vs2015u3
 VSWINPATH="$(cd ${MSVC_DIR} && pwd)"
 
 echo vswinpath ${VSWINPATH}
 
 export WINDOWSSDKDIR="${VSWINPATH}/SDK"
--- a/taskcluster/scripts/misc/build-clang-windows-helper64.sh
+++ b/taskcluster/scripts/misc/build-clang-windows-helper64.sh
@@ -8,17 +8,17 @@ set -x -e -v
 export TOOLTOOL_CACHE
 
 TOOLTOOL_AUTH_FILE=/c/builds/relengapi.tok
 if [ ! -e ${TOOLTOOL_AUTH_FILE} ]; then
     echo cannot find ${TOOLTOOL_AUTH_FILE}
     exit 1
 fi
 
-./build/src/mach artifact toolchain -v --authentication-file="${TOOLTOOL_AUTH_FILE}" --tooltool-manifest "build/src/${TOOLTOOL_MANIFEST}"${TOOLTOOL_CACHE:+ --cache-dir ${TOOLTOOL_CACHE}}
+./build/src/mach artifact toolchain -v --authentication-file="${TOOLTOOL_AUTH_FILE}" --tooltool-manifest "build/src/${TOOLTOOL_MANIFEST}"${TOOLTOOL_CACHE:+ --cache-dir ${TOOLTOOL_CACHE}}${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}}
 
 # Set up all the Visual Studio paths.
 MSVC_DIR=vs2015u3
 VSWINPATH="$(cd ${MSVC_DIR} && pwd)"
 
 echo vswinpath ${VSWINPATH}
 
 export WINDOWSSDKDIR="${VSWINPATH}/SDK"
--- a/taskcluster/scripts/misc/tooltool-download.sh
+++ b/taskcluster/scripts/misc/tooltool-download.sh
@@ -21,11 +21,11 @@ fi
 if [ -n "$RELENGAPI_PORT" ]; then
     # When the worker has the relengapi proxy setup, use it.
     TOOLTOOL_DL_FLAGS="${TOOLTOOL_DL_FLAGS=} --tooltool-url=http://relengapi/tooltool/"
 fi
 
 : TOOLTOOL_CACHE                ${TOOLTOOL_CACHE:=/home/worker/tooltool-cache}
 export TOOLTOOL_CACHE
 
-./mach artifact toolchain -v${TOOLTOOL_DL_FLAGS} --tooltool-manifest "${TOOLTOOL_MANIFEST}"${TOOLTOOL_CACHE:+ --cache-dir ${TOOLTOOL_CACHE}} --retry 5
+./mach artifact toolchain -v${TOOLTOOL_DL_FLAGS} --tooltool-manifest "${TOOLTOOL_MANIFEST}"${TOOLTOOL_CACHE:+ --cache-dir ${TOOLTOOL_CACHE}} --retry 5${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}}
 
 cd $OLDPWD
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -1135,16 +1135,19 @@ or run without that action (ie: --no-{ac
             c['tooltool_url'],
         ]
         auth_file = self._get_tooltool_auth_file()
         if auth_file:
             cmd.extend(['--authentication-file', auth_file])
         cache = c['env'].get('TOOLTOOL_CACHE')
         if cache:
             cmd.extend(['--cache-dir', cache])
+        toolchains = os.environ.get('MOZ_TOOLCHAINS')
+        if toolchains:
+            cmd.extend(toolchains.split())
         self.info(str(cmd))
         self.run_command_m(cmd, cwd=dirs['abs_src_dir'], halt_on_failure=True,
                            env=env)
 
     def query_revision(self, source_path=None):
         """ returns the revision of the build
 
          first will look for it in buildbot_properties and then in
--- a/testing/mozharness/mozharness/mozilla/tooltool.py
+++ b/testing/mozharness/mozharness/mozilla/tooltool.py
@@ -87,16 +87,19 @@ class TooltoolMixin(object):
         if self.topsrcdir:
             cmd.extend(['--tooltool-manifest', manifest])
         else:
             cmd.extend(['fetch', '-m', manifest, '-o'])
 
         if cache:
             cmd.extend(['--cache-dir' if self.topsrcdir else '-c', cache])
 
+        toolchains = os.environ.get('MOZ_TOOLCHAINS')
+        if toolchains:
+            cmd.extend(toolchains.split())
         # when mock is enabled run tooltool in mock. We can't use
         # run_command_m in all cases because it won't exist unless
         # MockMixin is used on the parent class
         if self.config.get('mock_target'):
             cmd_runner = self.run_command_m
         else:
             cmd_runner = self.run_command
 
--- a/testing/mozharness/scripts/desktop_l10n.py
+++ b/testing/mozharness/scripts/desktop_l10n.py
@@ -1081,16 +1081,19 @@ class DesktopSingleLocale(LocalesMixin, 
             config['tooltool_url'],
         ]
         auth_file = self._get_tooltool_auth_file()
         if auth_file and os.path.exists(auth_file):
             cmd.extend(['--authentication-file', auth_file])
         cache = config['bootstrap_env'].get('TOOLTOOL_CACHE')
         if cache:
             cmd.extend(['--cache-dir', cache])
+        toolchains = os.environ.get('MOZ_TOOLCHAINS')
+        if toolchains:
+            cmd.extend(toolchains.split())
         self.info(str(cmd))
         self.run_command(cmd, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True,
                          env=env)
 
     def funsize_props(self):
         """Set buildbot properties required to trigger funsize tasks
          responsible to generate partial updates for successfully generated locales"""
         locales = self.query_locales()