Bug 1356683 - Use new tooltool wrapper in mozharness. r?chmanchester draft
authorMike Hommey <mh+mozilla@glandium.org>
Wed, 12 Apr 2017 17:48:18 +0900
changeset 563040 422d2ea930f6daf35973b9cea1341ca913dd8f2b
parent 563039 37f9f677fd50bb17eb10df45312a5c189c005ee2
child 563041 88861ddb72437f30b014277eca9f28f7a4e0c923
push id54195
push userbmo:mh+mozilla@glandium.org
push dateFri, 14 Apr 2017 21:59:19 +0000
reviewerschmanchester
bugs1356683
milestone55.0a1
Bug 1356683 - Use new tooltool wrapper in mozharness. r?chmanchester
testing/mozharness/mozharness/mozilla/building/buildbase.py
testing/mozharness/mozharness/mozilla/tooltool.py
testing/mozharness/scripts/desktop_l10n.py
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -1116,46 +1116,51 @@ or run without that action (ie: --no-{ac
         # if the file doesn't exist, don't pass it to tooltool (it will just
         # fail).  In taskcluster, this will work OK as the relengapi-proxy will
         # take care of auth.  Everywhere else, we'll get auth failures if
         # necessary.
         if os.path.exists(fn):
             return fn
 
     def _run_tooltool(self):
+        env = self.query_build_env()
+        env.update(self.query_mach_build_env())
+
         self._assert_cfg_valid_for_action(
-            ['tooltool_script', 'tooltool_bootstrap', 'tooltool_url'],
+            ['tooltool_script', 'tooltool_url'],
             'build'
         )
         c = self.config
         dirs = self.query_abs_dirs()
         if not c.get('tooltool_manifest_src'):
             return self.warning(ERROR_MSGS['tooltool_manifest_undetermined'])
-        fetch_script_path = os.path.join(dirs['abs_tools_dir'],
-                                         'scripts',
-                                         'tooltool',
-                                         'tooltool_wrapper.sh')
         tooltool_manifest_path = os.path.join(dirs['abs_src_dir'],
                                               c['tooltool_manifest_src'])
+        python = self.query_exe('python2.7')
         cmd = [
-            'sh',
-            fetch_script_path,
+            python, '-u',
+            os.path.join(dirs['abs_src_dir'], 'mach'),
+            'artifact',
+            'toolchain',
+            '-v',
+            '--retry', '4',
+            '--tooltool-manifest',
             tooltool_manifest_path,
+            '--tooltool-url',
             c['tooltool_url'],
-            c['tooltool_bootstrap'],
         ]
-        cmd.extend(c['tooltool_script'])
         auth_file = self._get_tooltool_auth_file()
         if auth_file:
             cmd.extend(['--authentication-file', auth_file])
         cache = c['env'].get('TOOLTOOL_CACHE')
         if cache:
-            cmd.extend(['-c', cache])
+            cmd.extend(['--cache-dir', cache])
         self.info(str(cmd))
-        self.run_command_m(cmd, cwd=dirs['abs_src_dir'], halt_on_failure=True)
+        self.run_command_m(cmd, cwd=dirs['abs_src_dir'], halt_on_failure=True,
+                           env=env)
 
     def query_revision(self, source_path=None):
         """ returns the revision of the build
 
          first will look for it in buildbot_properties and then in
          buildbot_config. Failing that, it will actually poll the source of
          the repo if it exists yet.
 
--- a/testing/mozharness/mozharness/mozilla/tooltool.py
+++ b/testing/mozharness/mozharness/mozilla/tooltool.py
@@ -45,19 +45,21 @@ class TooltoolMixin(object):
             return fn
 
     def tooltool_fetch(self, manifest,
                        output_dir=None, privileged=False, cache=None):
         """docstring for tooltool_fetch"""
         # Use vendored tooltool.py if available.
         if self.topsrcdir:
             cmd = [
-                sys.executable,
-                os.path.join(self.topsrcdir, 'python', 'mozbuild', 'mozbuild',
-                                'action', 'tooltool.py')
+                sys.executable, '-u',
+                os.path.join(self.topsrcdir, 'mach'),
+                'artifact',
+                'toolchain',
+                '-v',
             ]
         elif self.config.get("download_tooltool"):
             cmd = [sys.executable, self._fetch_tooltool_py()]
         else:
             cmd = self.query_exe('tooltool.py', return_type='list')
 
         # get the tooltool servers from configuration
         default_urls = self.config.get('tooltool_servers', TOOLTOOL_SERVERS)
@@ -67,27 +69,30 @@ class TooltoolMixin(object):
             return url if url.endswith('/') else (url + '/')
         default_urls = [add_slash(u) for u in default_urls]
 
         # proxxy-ify
         proxxy = Proxxy(self.config, self.log_obj)
         proxxy_urls = proxxy.get_proxies_and_urls(default_urls)
 
         for proxyied_url in proxxy_urls:
-            cmd.extend(['--url', proxyied_url])
+            cmd.extend(['--tooltool-url' if self.topsrcdir else '--url', proxyied_url])
 
         # handle authentication file, if given
         auth_file = self._get_auth_file()
         if auth_file and os.path.exists(auth_file):
             cmd.extend(['--authentication-file', auth_file])
 
-        cmd.extend(['fetch', '-m', manifest, '-o'])
+        if self.topsrcdir:
+            cmd.extend(['--tooltool-manifest', manifest])
+        else:
+            cmd.extend(['fetch', '-m', manifest, '-o'])
 
         if cache:
-            cmd.extend(['-c', cache])
+            cmd.extend(['--cache-dir' if self.topsrcdir else '-c', cache])
 
         # when mock is enabled run tooltool in mock. We can't use
         # run_command_m in all cases because it won't exist unless
         # MockMixin is used on the parent class
         if self.config.get('mock_target'):
             cmd_runner = self.run_command_m
         else:
             cmd_runner = self.run_command
--- a/testing/mozharness/scripts/desktop_l10n.py
+++ b/testing/mozharness/scripts/desktop_l10n.py
@@ -1047,40 +1047,45 @@ class DesktopSingleLocale(LocalesMixin, 
         # if the file doesn't exist, don't pass it to tooltool (it will just
         # fail).  In taskcluster, this will work OK as the relengapi-proxy will
         # take care of auth.  Everywhere else, we'll get auth failures if
         # necessary.
         if os.path.exists(fn):
             return fn
 
     def _run_tooltool(self):
+        env = self.query_bootstrap_env()
         config = self.config
         dirs = self.query_abs_dirs()
         if not config.get('tooltool_manifest_src'):
             return self.warning(ERROR_MSGS['tooltool_manifest_undetermined'])
-        fetch_script_path = os.path.join(dirs['abs_tools_dir'],
-                                         'scripts/tooltool/tooltool_wrapper.sh')
         tooltool_manifest_path = os.path.join(dirs['abs_mozilla_dir'],
                                               config['tooltool_manifest_src'])
+        python = self.query_exe('python2.7')
         cmd = [
-            'sh',
-            fetch_script_path,
+            python, '-u',
+            os.path.join(dirs['abs_mozilla_dir'], 'mach'),
+            'artifact',
+            'toolchain',
+            '-v',
+            '--retry', '4',
+            '--tooltool-manifest',
             tooltool_manifest_path,
+            '--tooltool-url',
             config['tooltool_url'],
-            config['tooltool_bootstrap'],
         ]
-        cmd.extend(config['tooltool_script'])
         auth_file = self._get_tooltool_auth_file()
         if auth_file and os.path.exists(auth_file):
             cmd.extend(['--authentication-file', auth_file])
         cache = config['bootstrap_env'].get('TOOLTOOL_CACHE')
         if cache:
-            cmd.extend(['-c', cache])
+            cmd.extend(['--cache-dir', cache])
         self.info(str(cmd))
-        self.run_command(cmd, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True)
+        self.run_command(cmd, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True,
+                         env=env)
 
     def funsize_props(self):
         """Set buildbot properties required to trigger funsize tasks
          responsible to generate partial updates for successfully generated locales"""
         locales = self.query_locales()
         funsize_info = {
             'locales': locales,
             'branch': self.config['branch'],