Bug 1284475 - migrate ToolTool blueprint to new codebase of relengapi r=KWierso
authorRok Garbas <rok@garbas.si>
Tue, 08 Aug 2017 10:55:52 +0200
changeset 376373 0eebae2f09777d3ab4df1ea8cc951541d81b7647
parent 376372 16b6c9d86e7f94256dfd86dd8c5f5e3d5cc597a1
child 376374 9995d46d468f2d43bf48c14feee78ff419229349
push id49437
push userkwierso@gmail.com
push dateWed, 23 Aug 2017 18:14:17 +0000
treeherderautoland@0eebae2f0977 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersKWierso
bugs1284475
milestone57.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1284475 - migrate ToolTool blueprint to new codebase of relengapi r=KWierso UPGRADE_NSS_RELEASE a=kaie MozReview-Commit-ID: 7CB1VYb8OJP
js/src/devtools/automation/winbuildenv.sh
python/mozboot/bin/bootstrap-msys2.vbs
python/mozbuild/mozbuild/action/tooltool.py
python/mozbuild/mozbuild/mach_commands.py
security/nss/automation/taskcluster/windows/setup.sh
taskcluster/docker/centos6-build/system-setup.sh
taskcluster/docker/recipes/common.sh
taskcluster/docker/recipes/ubuntu1204-test-system-setup.sh
taskcluster/scripts/builder/build-haz-linux.sh
taskcluster/scripts/builder/sm-tooltool-config.sh
testing/mozharness/configs/developer_config.py
testing/mozharness/configs/single_locale/win32.py
testing/mozharness/configs/single_locale/win32_devedition.py
testing/mozharness/configs/single_locale/win64.py
testing/mozharness/configs/single_locale/win64_devedition.py
testing/mozharness/mozharness/mozilla/testing/codecoverage.py
testing/mozharness/mozharness/mozilla/tooltool.py
testing/mozharness/scripts/fx_desktop_build.py
tools/mach_commands.py
--- a/js/src/devtools/automation/winbuildenv.sh
+++ b/js/src/devtools/automation/winbuildenv.sh
@@ -25,12 +25,12 @@ else
   . $topsrcdir/build/win32/mozconfig.vs-latest
 fi
 
 # PATH also needs to point to mozmake.exe, which can come from either
 # newer mozilla-build or tooltool.
 if ! which mozmake 2>/dev/null; then
     export PATH="$PATH:$SOURCE/.."
     if ! which mozmake 2>/dev/null; then
-  TT_SERVER=${TT_SERVER:-https://api.pub.build.mozilla.org/tooltool/}
+  TT_SERVER=${TT_SERVER:-https://tooltool.mozilla-releng.net/}
   ( cd $SOURCE/..; $SOURCE/mach artifact toolchain -v --tooltool-manifest $SOURCE/browser/config/tooltool-manifests/${platform:-win32}/releng.manifest --tooltool-url $TT_SERVER --retry 4${TOOLTOOL_CACHE:+ --cache-dir ${TOOLTOOL_CACHE}}${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}})
     fi
 fi
--- a/python/mozboot/bin/bootstrap-msys2.vbs
+++ b/python/mozboot/bin/bootstrap-msys2.vbs
@@ -48,17 +48,17 @@ If NOT fso.FolderExists(installPath) The
 End If
 If NOT fso.FolderExists(installPath) Then
     MsgBox("Failed to create folder. Do you have permission to install in this directory?")
     WScript.Quit 1
 End If
 
 On Error Resume Next
 ' Download and move MSYS2 into the right place
-Download "https://api.pub.build.mozilla.org/tooltool/sha512/f93a685c8a10abbd349cbef5306441ba235c4cbfba1cc000299e11b58f258e9953cbe23463515407925eeca94c3f5d8e5f637c95be387e620845efa43cdcb0c0", "msys2.zip"
+Download "https://tooltool.mozilla-releng.net/sha512/f93a685c8a10abbd349cbef5306441ba235c4cbfba1cc000299e11b58f258e9953cbe23463515407925eeca94c3f5d8e5f637c95be387e620845efa43cdcb0c0", "msys2.zip"
 Set FilesInZip = appShell.NameSpace(fso.GetAbsolutePathName("msys2.zip")).Items()
 appShell.NameSpace(msysPath).CopyHere(FilesInZip)
 ' MSYS2 archive doesn't have tmp directory...
 fso.CreateFolder(fso.BuildPath(msysPath, "tmp"))
 fso.DeleteFile("msys2.zip")
 If Err.Number <> 0 Then
     MsgBox("Error downloading and installing MSYS2. Make sure you have internet connection. If you think this is a bug, please file one in Bugzilla https://bugzilla.mozilla.org/enter_bug.cgi?product=Core&component=Build%20Config")
     WScript.Quit 1
--- a/python/mozbuild/mozbuild/action/tooltool.py
+++ b/python/mozbuild/mozbuild/action/tooltool.py
@@ -1,9 +1,9 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 
 # tooltool is a lookaside cache implemented in Python
 # Copyright (C) 2011 John H. Ford <john@johnford.info>
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
 # as published by the Free Software Foundation version 2
 #
@@ -994,17 +994,17 @@ def main(argv, _skip_logging=False):
                       help='Use the RelengAPI token found in the given file to '
                            'authenticate to the RelengAPI server.',
                       dest='auth_file')
 
     (options_obj, args) = parser.parse_args(argv[1:])
 
     # default the options list if not provided
     if not options_obj.base_url:
-        options_obj.base_url = ['https://api.pub.build.mozilla.org/tooltool/']
+        options_obj.base_url = ['https://tooltool.mozilla-releng.net/']
 
     # ensure all URLs have a trailing slash
     def add_slash(url):
         return url if url.endswith('/') else (url + '/')
     options_obj.base_url = [add_slash(u) for u in options_obj.base_url]
 
     # expand ~ in --authentication-file
     if options_obj.auth_file:
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1797,17 +1797,17 @@ class PackageFrontend(MachCommandBase):
             logging.getLogger('redo').addHandler(
                 self.log_manager.terminal_handler)
             self.log_manager.terminal_handler.addFilter(
                 self.log_manager.structured_filter)
         if not cache_dir:
             cache_dir = os.path.join(self._mach_context.state_dir, 'toolchains')
 
         tooltool_url = (tooltool_url or
-                        'https://api.pub.build.mozilla.org/tooltool').rstrip('/')
+                        'https://tooltool.mozilla-releng.net').rstrip('/')
 
         cache = ArtifactCache(cache_dir=cache_dir, log=self.log,
                               skip_cache=skip_cache)
 
         if authentication_file:
             with open(authentication_file, 'rb') as f:
                 token = f.read().strip()
 
--- a/security/nss/automation/taskcluster/windows/setup.sh
+++ b/security/nss/automation/taskcluster/windows/setup.sh
@@ -18,9 +18,9 @@ hg_clone() {
         sleep $i
         hg clone -r "$rev" "$repo" "$dir" && return
         rm -rf "$dir"
     done
     exit 1
 }
 
 hg_clone https://hg.mozilla.org/build/tools tools default
-tools/scripts/tooltool/tooltool_wrapper.sh $(dirname $0)/releng.manifest https://api.pub.build.mozilla.org/tooltool/ non-existant-file.sh /c/mozilla-build/python/python.exe /c/builds/tooltool.py --authentication-file /c/builds/relengapi.tok -c /c/builds/tooltool_cache
+tools/scripts/tooltool/tooltool_wrapper.sh $(dirname $0)/releng.manifest https://tooltool.mozilla-releng.net/ non-existant-file.sh /c/mozilla-build/python/python.exe /c/builds/tooltool.py --authentication-file /c/builds/relengapi.tok -c /c/builds/tooltool_cache
--- a/taskcluster/docker/centos6-build/system-setup.sh
+++ b/taskcluster/docker/centos6-build/system-setup.sh
@@ -279,17 +279,17 @@ mkdir $BUILD
 # for the builds below, there's no sense using ccache
 export CCACHE_DISABLE=1
 
 cd $BUILD
 curl https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py > tooltool.py
 
 tooltool_fetch() {
     cat >manifest.tt
-    python $BUILD/tooltool.py fetch
+    python2.7 $BUILD/tooltool.py fetch
     rm manifest.tt
 }
 
 # For a few packges, we want to run the very latest, which is hard to find for
 # stable old CentOS 6.  Custom yum repostiories are cumbersome and can cause
 # unhappy failures when they contain multiple versions of the same package.  So
 # we either build from source or install an RPM from tooltool (the former being
 # the preferred solution for transparency).  Each of these source files was
--- a/taskcluster/docker/recipes/common.sh
+++ b/taskcluster/docker/recipes/common.sh
@@ -1,10 +1,10 @@
 #!/bin/bash
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 tooltool_fetch() {
     cat >manifest.tt
-    python /setup/tooltool.py fetch
+    python2.7 /setup/tooltool.py fetch
     rm manifest.tt
 }
--- a/taskcluster/docker/recipes/ubuntu1204-test-system-setup.sh
+++ b/taskcluster/docker/recipes/ubuntu1204-test-system-setup.sh
@@ -121,17 +121,17 @@ apt-get update
 # This allows ubuntu-desktop to be installed without human interaction
 export DEBIAN_FRONTEND=noninteractive
 apt-get install -y --force-yes ${apt_packages[@]}
 
 dpkg-reconfigure locales
 
 tooltool_fetch() {
     cat >manifest.tt
-    python /setup/tooltool.py fetch
+    python2.7 /setup/tooltool.py fetch
     rm manifest.tt
 }
 
 . /tmp/install-mercurial.sh
 
 # install peep
 tooltool_fetch <<'EOF'
 [
--- a/taskcluster/scripts/builder/build-haz-linux.sh
+++ b/taskcluster/scripts/builder/build-haz-linux.sh
@@ -48,17 +48,17 @@ GECKO_DIR=$( cd "$GECKO_DIR" && pwd )
 # Directory to populate with tooltool-installed tools
 export TOOLTOOL_DIR="$WORKSPACE"
 
 # Directory to hold the (useless) object files generated by the analysis.
 export MOZ_OBJDIR="$WORKSPACE/obj-analyzed"
 mkdir -p "$MOZ_OBJDIR"
 
 if [ -n "$DO_TOOLTOOL" ]; then
-  ( cd $TOOLTOOL_DIR; $GECKO_DIR/mach artifact toolchain -v --tooltool-url https://api.pub.build.mozilla.org/tooltool/ --tooltool-manifest $GECKO_DIR/$TOOLTOOL_MANIFEST --cache-dir $TOOLTOOL_CACHE${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}} )
+  ( cd $TOOLTOOL_DIR; $GECKO_DIR/mach artifact toolchain -v --tooltool-url https://tooltool.mozilla-releng.net/ --tooltool-manifest $GECKO_DIR/$TOOLTOOL_MANIFEST --cache-dir $TOOLTOOL_CACHE${MOZ_TOOLCHAINS:+ ${MOZ_TOOLCHAINS}} )
 fi
 
 export NO_MERCURIAL_SETUP_CHECK=1
 
 if [[ "$PROJECT" = "browser" ]]; then (
     cd "$WORKSPACE"
     set "$WORKSPACE"
     . setup-ccache.sh
--- a/taskcluster/scripts/builder/sm-tooltool-config.sh
+++ b/taskcluster/scripts/builder/sm-tooltool-config.sh
@@ -1,13 +1,13 @@
 #!/bin/bash
 
 set -xe
 
-: ${TOOLTOOL_SERVER:=https://api.pub.build.mozilla.org/tooltool/}
+: ${TOOLTOOL_SERVER:=https://tooltool.mozilla-releng.net/}
 : ${SPIDERMONKEY_VARIANT:=plain}
 : ${UPLOAD_DIR:=$HOME/artifacts/}
 : ${WORK:=$HOME/workspace}
 : ${SRCDIR:=$WORK/build/src}
 
 mkdir -p $WORK
 cd $WORK
 
--- a/testing/mozharness/configs/developer_config.py
+++ b/testing/mozharness/configs/developer_config.py
@@ -41,10 +41,10 @@ config = {
     "tooltool_cache": os.path.join(LOCAL_WORKDIR, "builds/tooltool_cache"),
     "tooltool_cache_path": os.path.join(LOCAL_WORKDIR, "builds/tooltool_cache"),
     "tooltool_authentication_file": os.path.join(LOCAL_WORKDIR, "relengapi.tok"),
 
     # VCS tools
     "gittool.py": 'http://hg.mozilla.org/build/puppet/raw-file/faaf5abd792e/modules/packages/files/gittool.py',
 
     # Android related
-    "host_utils_url": "https://api.pub.build.mozilla.org/tooltool/sha512/372c89f9dccaf5ee3b9d35fd1cfeb089e1e5db3ff1c04e35aa3adc8800bc61a2ae10e321f37ae7bab20b56e60941f91bb003bcb22035902a73d70872e7bd3282",
+    "host_utils_url": "https://tooltool.mozilla-releng.net/sha512/372c89f9dccaf5ee3b9d35fd1cfeb089e1e5db3ff1c04e35aa3adc8800bc61a2ae10e321f37ae7bab20b56e60941f91bb003bcb22035902a73d70872e7bd3282",
 }
--- a/testing/mozharness/configs/single_locale/win32.py
+++ b/testing/mozharness/configs/single_locale/win32.py
@@ -24,17 +24,17 @@ config = {
     },
     "ssh_key_dir": "~/.ssh",
     "log_name": "single_locale",
     "objdir": "obj-firefox",
     "js_src_dir": "js/src",
     "vcs_share_base": "c:/builds/hg-shared",
 
     # tooltool
-    'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
+    'tooltool_url': 'https://tooltool.mozilla-releng.net/',
     'tooltool_script': [sys.executable,
                         'C:/mozilla-build/tooltool.py'],
     'tooltool_manifest_src': 'browser/config/tooltool-manifests/win32/l10n.manifest',
     # balrog credential file:
     'balrog_credentials_file': 'oauth.txt',
 
     # l10n
     "ignore_locales": ["en-US", "ja-JP-mac"],
--- a/testing/mozharness/configs/single_locale/win32_devedition.py
+++ b/testing/mozharness/configs/single_locale/win32_devedition.py
@@ -24,17 +24,17 @@ config = {
     },
     "ssh_key_dir": "~/.ssh",
     "log_name": "single_locale",
     "objdir": "obj-firefox",
     "js_src_dir": "js/src",
     "vcs_share_base": "c:/builds/hg-shared",
 
     # tooltool
-    'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
+    'tooltool_url': 'https://tooltool.mozilla-releng.net/',
     'tooltool_script': [sys.executable,
                         'C:/mozilla-build/tooltool.py'],
     'tooltool_manifest_src': 'browser/config/tooltool-manifests/win32/l10n.manifest',
     # balrog credential file:
     'balrog_credentials_file': 'oauth.txt',
 
     # l10n
     "ignore_locales": ["en-US", "ja-JP-mac"],
--- a/testing/mozharness/configs/single_locale/win64.py
+++ b/testing/mozharness/configs/single_locale/win64.py
@@ -24,17 +24,17 @@ config = {
     },
     "ssh_key_dir": "~/.ssh",
     "log_name": "single_locale",
     "objdir": "obj-firefox",
     "js_src_dir": "js/src",
     "vcs_share_base": "c:/builds/hg-shared",
 
     # tooltool
-    'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
+    'tooltool_url': 'https://tooltool.mozilla-releng.net/',
     'tooltool_script': [sys.executable,
                         'C:/mozilla-build/tooltool.py'],
     'tooltool_manifest_src': 'browser/config/tooltool-manifests/win64/l10n.manifest',
     # balrog credential file:
     'balrog_credentials_file': 'oauth.txt',
 
     # l10n
     "ignore_locales": ["en-US", "ja-JP-mac"],
--- a/testing/mozharness/configs/single_locale/win64_devedition.py
+++ b/testing/mozharness/configs/single_locale/win64_devedition.py
@@ -24,17 +24,17 @@ config = {
     },
     "ssh_key_dir": "~/.ssh",
     "log_name": "single_locale",
     "objdir": "obj-firefox",
     "js_src_dir": "js/src",
     "vcs_share_base": "c:/builds/hg-shared",
 
     # tooltool
-    'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
+    'tooltool_url': 'https://tooltool.mozilla-releng.net/',
     'tooltool_script': [sys.executable,
                         'C:/mozilla-build/tooltool.py'],
     'tooltool_manifest_src': 'browser/config/tooltool-manifests/win64/l10n.manifest',
     # balrog credential file:
     'balrog_credentials_file': 'oauth.txt',
 
     # l10n
     "ignore_locales": ["en-US", "ja-JP-mac"],
--- a/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
+++ b/testing/mozharness/mozharness/mozilla/testing/codecoverage.py
@@ -76,17 +76,17 @@ class CodeCoverageMixin(object):
 
         # Create the grcov directory, get the tooltool manifest, and finally
         # download and unpack the grcov binary.
         self.grcov_dir = tempfile.mkdtemp()
         manifest = os.path.join(dirs.get('abs_test_install_dir', os.path.join(dirs['abs_work_dir'], 'tests')), \
             'config/tooltool-manifests/linux64/ccov.manifest')
 
         tooltool_path = self._fetch_tooltool_py()
-        cmd = [tooltool_path, '--url', 'https://api.pub.build.mozilla.org/tooltool/', 'fetch', \
+        cmd = [tooltool_path, '--url', 'https://tooltool.mozilla-releng.net/', 'fetch', \
             '-m', manifest, '-o', '-c', '/home/worker/tooltool-cache']
         self.run_command(cmd, cwd=self.grcov_dir)
         self.run_command(['tar', '-jxvf', os.path.join(self.grcov_dir, 'grcov-linux-standalone-x86_64.tar.bz2'), \
             '-C', self.grcov_dir], cwd=self.grcov_dir)
 
     @PostScriptAction('run-tests')
     def _package_coverage_data(self, action, success=None):
         if not self.code_coverage_enabled:
--- a/testing/mozharness/mozharness/mozilla/tooltool.py
+++ b/testing/mozharness/mozharness/mozilla/tooltool.py
@@ -10,17 +10,17 @@ TooltoolErrorList = PythonErrorList + [{
     'substr': 'ERROR - ', 'level': ERROR
 }]
 
 
 TOOLTOOL_PY_URL = \
     "https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py"
 
 TOOLTOOL_SERVERS = [
-    'https://api.pub.build.mozilla.org/tooltool/',
+    'https://tooltool.mozilla-releng.net/',
 ]
 
 
 class TooltoolMixin(object):
     """Mixin class for handling tooltool manifests.
     To use a tooltool server other than the Mozilla server, override
     config['tooltool_servers'].  To specify a different authentication
     file than that used in releng automation,override
--- a/testing/mozharness/scripts/fx_desktop_build.py
+++ b/testing/mozharness/scripts/fx_desktop_build.py
@@ -61,17 +61,17 @@ class FxDesktopBuild(BuildScript, TryToo
                 'taskcluster_credentials_file': 'oauth.txt',
                 'periodic_clobber': 168,
                 # hg tool stuff
                 "tools_repo": "https://hg.mozilla.org/build/tools",
                 # Seed all clones with mozilla-unified. This ensures subsequent
                 # jobs have a minimal `hg pull`.
                 "clone_upstream_url": "https://hg.mozilla.org/mozilla-unified",
                 "repo_base": "https://hg.mozilla.org",
-                'tooltool_url': 'https://api.pub.build.mozilla.org/tooltool/',
+                'tooltool_url': 'https://tooltool.mozilla-releng.net/',
                 "graph_selector": "/server/collect.cgi",
                 # only used for make uploadsymbols
                 'old_packages': [
                     "%(objdir)s/dist/firefox-*",
                     "%(objdir)s/dist/fennec*",
                     "%(objdir)s/dist/seamonkey*",
                     "%(objdir)s/dist/thunderbird*",
                     "%(objdir)s/dist/install/sea/*.exe"
--- a/tools/mach_commands.py
+++ b/tools/mach_commands.py
@@ -223,17 +223,17 @@ class FormatProvider(MachCommandBase):
             "Linux": "e6da4f6df074bfb15caefcf7767eb5670c02bb4768ba86ae4ab6b35235b53db012900a4f9e9a950ee140158a19532a71f21b986f511826bebc16f2ef83984e57",  # noqa: E501
             "Darwin": "18000940a11e5ab0c1fe950d4360292216c8e963dd708679c4c5fb8cc845f5919cef3f58a7e092555b8ea6b8d8a809d66153ea6d1e7c226a2c4f2b0b7ad1b2f3",  # noqa: E501
             "python_script": "34b6934a48a263ea3f88d48c2981d61ae6698823cfa689b9b0c8a607c224437ca0b9fdd434d260bd790d52a98455e2c2e2c745490d327ba84b4e22b7bb55b757",  # noqa: E501
         }
 
         target = os.path.join(self._mach_context.state_dir, os.path.basename(root))
 
         if not os.path.exists(target):
-            tooltool_url = "https://api.pub.build.mozilla.org/tooltool/sha512/"
+            tooltool_url = "https://tooltool.mozilla-releng.net/sha512/"
             if self.prompt and raw_input("Download clang-format executables from {0} (yN)? ".format(tooltool_url)).lower() != 'y':  # noqa: E501,F821
                 print("Download aborted.")
                 return None
             self.prompt = False
             plat = platform.system()
             if python_script:
                 # We want to download the python script (clang-format-diff)
                 dl = bin_sha["python_script"]