Bug 1356683 - Use new tooltool wrapper in mozharness. r=chmanchester
authorMike Hommey <mh+mozilla@glandium.org>
Wed, 12 Apr 2017 17:48:18 +0900
changeset 357183 d51d444bd5b52eef6b71b6dad30e0864aeafdd17
parent 357182 35bb80a39be676fa3f0854a867d15dc96868bb91
child 357184 b4a1bde0208564e61df1a330b8db73421587f906
push id31783
push usercbook@mozilla.com
push dateTue, 09 May 2017 12:03:48 +0000
treeherdermozilla-central@b0ff0c5c0a35 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschmanchester
bugs1356683
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1356683 - Use new tooltool wrapper in mozharness. r=chmanchester
testing/mozharness/mozharness/mozilla/building/buildbase.py
testing/mozharness/mozharness/mozilla/tooltool.py
testing/mozharness/scripts/desktop_l10n.py
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -1117,46 +1117,51 @@ or run without that action (ie: --no-{ac
         # if the file doesn't exist, don't pass it to tooltool (it will just
         # fail).  In taskcluster, this will work OK as the relengapi-proxy will
         # take care of auth.  Everywhere else, we'll get auth failures if
         # necessary.
         if os.path.exists(fn):
             return fn
 
     def _run_tooltool(self):
+        env = self.query_build_env()
+        env.update(self.query_mach_build_env())
+
         self._assert_cfg_valid_for_action(
-            ['tooltool_script', 'tooltool_bootstrap', 'tooltool_url'],
+            ['tooltool_script', 'tooltool_url'],
             'build'
         )
         c = self.config
         dirs = self.query_abs_dirs()
         if not c.get('tooltool_manifest_src'):
             return self.warning(ERROR_MSGS['tooltool_manifest_undetermined'])
-        fetch_script_path = os.path.join(dirs['abs_tools_dir'],
-                                         'scripts',
-                                         'tooltool',
-                                         'tooltool_wrapper.sh')
         tooltool_manifest_path = os.path.join(dirs['abs_src_dir'],
                                               c['tooltool_manifest_src'])
+        python = self.query_exe('python2.7')
         cmd = [
-            'sh',
-            fetch_script_path,
+            python, '-u',
+            os.path.join(dirs['abs_src_dir'], 'mach'),
+            'artifact',
+            'toolchain',
+            '-v',
+            '--retry', '4',
+            '--tooltool-manifest',
             tooltool_manifest_path,
+            '--tooltool-url',
             c['tooltool_url'],
-            c['tooltool_bootstrap'],
         ]
-        cmd.extend(c['tooltool_script'])
         auth_file = self._get_tooltool_auth_file()
         if auth_file:
             cmd.extend(['--authentication-file', auth_file])
         cache = c['env'].get('TOOLTOOL_CACHE')
         if cache:
-            cmd.extend(['-c', cache])
+            cmd.extend(['--cache-dir', cache])
         self.info(str(cmd))
-        self.run_command_m(cmd, cwd=dirs['abs_src_dir'], halt_on_failure=True)
+        self.run_command_m(cmd, cwd=dirs['abs_src_dir'], halt_on_failure=True,
+                           env=env)
 
     def query_revision(self, source_path=None):
         """ returns the revision of the build
 
          first will look for it in buildbot_properties and then in
          buildbot_config. Failing that, it will actually poll the source of
          the repo if it exists yet.
 
--- a/testing/mozharness/mozharness/mozilla/tooltool.py
+++ b/testing/mozharness/mozharness/mozilla/tooltool.py
@@ -48,19 +48,21 @@ class TooltoolMixin(object):
                        output_dir=None, privileged=False, cache=None):
         """docstring for tooltool_fetch"""
         for d in (output_dir, cache):
             if d is not None and not os.path.exists(d):
                 self.mkdir_p(d)
         # Use vendored tooltool.py if available.
         if self.topsrcdir:
             cmd = [
-                sys.executable,
-                os.path.join(self.topsrcdir, 'python', 'mozbuild', 'mozbuild',
-                                'action', 'tooltool.py')
+                sys.executable, '-u',
+                os.path.join(self.topsrcdir, 'mach'),
+                'artifact',
+                'toolchain',
+                '-v',
             ]
         elif self.config.get("download_tooltool"):
             cmd = [sys.executable, self._fetch_tooltool_py()]
         else:
             cmd = self.query_exe('tooltool.py', return_type='list')
 
         # get the tooltool servers from configuration
         default_urls = self.config.get('tooltool_servers', TOOLTOOL_SERVERS)
@@ -70,27 +72,30 @@ class TooltoolMixin(object):
             return url if url.endswith('/') else (url + '/')
         default_urls = [add_slash(u) for u in default_urls]
 
         # proxxy-ify
         proxxy = Proxxy(self.config, self.log_obj)
         proxxy_urls = proxxy.get_proxies_and_urls(default_urls)
 
         for proxyied_url in proxxy_urls:
-            cmd.extend(['--url', proxyied_url])
+            cmd.extend(['--tooltool-url' if self.topsrcdir else '--url', proxyied_url])
 
         # handle authentication file, if given
         auth_file = self._get_auth_file()
         if auth_file and os.path.exists(auth_file):
             cmd.extend(['--authentication-file', auth_file])
 
-        cmd.extend(['fetch', '-m', manifest, '-o'])
+        if self.topsrcdir:
+            cmd.extend(['--tooltool-manifest', manifest])
+        else:
+            cmd.extend(['fetch', '-m', manifest, '-o'])
 
         if cache:
-            cmd.extend(['-c', cache])
+            cmd.extend(['--cache-dir' if self.topsrcdir else '-c', cache])
 
         # when mock is enabled run tooltool in mock. We can't use
         # run_command_m in all cases because it won't exist unless
         # MockMixin is used on the parent class
         if self.config.get('mock_target'):
             cmd_runner = self.run_command_m
         else:
             cmd_runner = self.run_command
--- a/testing/mozharness/scripts/desktop_l10n.py
+++ b/testing/mozharness/scripts/desktop_l10n.py
@@ -1028,40 +1028,45 @@ class DesktopSingleLocale(LocalesMixin, 
         # if the file doesn't exist, don't pass it to tooltool (it will just
         # fail).  In taskcluster, this will work OK as the relengapi-proxy will
         # take care of auth.  Everywhere else, we'll get auth failures if
         # necessary.
         if os.path.exists(fn):
             return fn
 
     def _run_tooltool(self):
+        env = self.query_bootstrap_env()
         config = self.config
         dirs = self.query_abs_dirs()
         if not config.get('tooltool_manifest_src'):
             return self.warning(ERROR_MSGS['tooltool_manifest_undetermined'])
-        fetch_script_path = os.path.join(dirs['abs_tools_dir'],
-                                         'scripts/tooltool/tooltool_wrapper.sh')
         tooltool_manifest_path = os.path.join(dirs['abs_mozilla_dir'],
                                               config['tooltool_manifest_src'])
+        python = self.query_exe('python2.7')
         cmd = [
-            'sh',
-            fetch_script_path,
+            python, '-u',
+            os.path.join(dirs['abs_mozilla_dir'], 'mach'),
+            'artifact',
+            'toolchain',
+            '-v',
+            '--retry', '4',
+            '--tooltool-manifest',
             tooltool_manifest_path,
+            '--tooltool-url',
             config['tooltool_url'],
-            config['tooltool_bootstrap'],
         ]
-        cmd.extend(config['tooltool_script'])
         auth_file = self._get_tooltool_auth_file()
         if auth_file and os.path.exists(auth_file):
             cmd.extend(['--authentication-file', auth_file])
         cache = config['bootstrap_env'].get('TOOLTOOL_CACHE')
         if cache:
-            cmd.extend(['-c', cache])
+            cmd.extend(['--cache-dir', cache])
         self.info(str(cmd))
-        self.run_command(cmd, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True)
+        self.run_command(cmd, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True,
+                         env=env)
 
     def funsize_props(self):
         """Set buildbot properties required to trigger funsize tasks
          responsible to generate partial updates for successfully generated locales"""
         locales = self.query_locales()
         funsize_info = {
             'locales': locales,
             'branch': self.config['branch'],