Bug 1170753: use the relengapi proxy for tooltool downloads; r=ted
authorDustin J. Mitchell <dustin@mozilla.com>
Thu, 02 Jul 2015 12:13:57 -0400 (2015-07-02)
changeset 4228 354aa7517304d97d3377df020b404b287be1b8bc
parent 4227 f59b5364984e8bfa147b321aa35d2b75b7dca52f
child 4229 593033fdb675a7e18a0c60ffc69af1c30490e44c
push id3367
push userdmitchell@mozilla.com
push dateMon, 06 Jul 2015 15:47:53 +0000 (2015-07-06)
reviewersted
bugs1170753
Bug 1170753: use the relengapi proxy for tooltool downloads; r=ted
configs/builds/build_pool_specifics.py
mozharness/mozilla/building/buildbase.py
mozharness/mozilla/tooltool.py
scripts/desktop_l10n.py
--- a/configs/builds/build_pool_specifics.py
+++ b/configs/builds/build_pool_specifics.py
@@ -34,10 +34,13 @@ config = {
         'stage_server': 'stage.mozilla.org',
         "sendchange_masters": ["buildbot-master81.build.mozilla.org:9301"],
         'taskcluster_index': 'index',
     },
     "taskcluster": {
         'graph_server': 'graphs.mozilla.org',
         'symbol_server_host': "symbolpush.mozilla.org",
         'stage_server': 'stage.mozilla.org',
+        # use the relengapi proxy to talk to tooltool
+        "tooltool_servers": ['http://relengapi/tooltool/'],
+        "tooltool_url": 'http://relengapi/tooltool/',
     },
 }
--- a/mozharness/mozilla/building/buildbase.py
+++ b/mozharness/mozilla/building/buildbase.py
@@ -1062,22 +1062,28 @@ or run without that action (ie: --no-{ac
                        "tree to use use. Please provide the path in your "
                        "config via 'src_mozconfig'")
 
     # TODO: replace with ToolToolMixin
     def _get_tooltool_auth_file(self):
         # set the default authentication file based on platform; this
         # corresponds to where puppet puts the token
         if 'tooltool_authentication_file' in self.config:
-            return self.config['tooltool_authentication_file']
+            fn = self.config['tooltool_authentication_file']
+        elif self._is_windows():
+            fn = r'c:\builds\relengapi.tok'
+        else:
+            fn = '/builds/relengapi.tok'
 
-        if self._is_windows():
-            return r'c:\builds\relengapi.tok'
-        else:
-            return '/builds/relengapi.tok'
+        # if the file doesn't exist, don't pass it to tooltool (it will just
+        # fail).  In taskcluster, this will work OK as the relengapi-proxy will
+        # take care of auth.  Everywhere else, we'll get auth failures if
+        # necessary.
+        if os.path.exists(fn):
+            return fn
 
     def _run_tooltool(self):
         self._assert_cfg_valid_for_action(
             ['tooltool_script', 'tooltool_bootstrap', 'tooltool_url'],
             'build'
         )
         c = self.config
         dirs = self.query_abs_dirs()
@@ -1090,17 +1096,19 @@ or run without that action (ie: --no-{ac
         cmd = [
             'sh',
             fetch_script_path,
             tooltool_manifest_path,
             c['tooltool_url'],
             c['tooltool_bootstrap'],
         ]
         cmd.extend(c['tooltool_script'])
-        cmd.extend(['--authentication-file', self._get_tooltool_auth_file()])
+        auth_file = self._get_tooltool_auth_file()
+        if auth_file:
+            cmd.extend(['--authentication-file', auth_file])
         self.info(str(cmd))
         self.run_command(cmd, cwd=dirs['abs_src_dir'], halt_on_failure=True)
 
     def query_revision(self, source_path=None):
         """ returns the revision of the build
 
          first will look for it in buildbot_properties and then in
          buildbot_config. Failing that, it will actually poll the source of
--- a/mozharness/mozilla/tooltool.py
+++ b/mozharness/mozilla/tooltool.py
@@ -28,22 +28,28 @@ class TooltoolMixin(object):
     file than that used in releng automation,override
     config['tooltool_authentication_file']; set it to None to not pass
     any authentication information (OK for public files)
     """
     def _get_auth_file(self):
         # set the default authentication file based on platform; this
         # corresponds to where puppet puts the token
         if 'tooltool_authentication_file' in self.config:
-            return self.config['tooltool_authentication_file']
+            fn = self.config['tooltool_authentication_file']
+        elif self._is_windows():
+            fn = r'c:\builds\relengapi.tok'
+        else:
+            fn = '/builds/relengapi.tok'
 
-        if self._is_windows():
-            return r'c:\builds\relengapi.tok'
-        else:
-            return '/builds/relengapi.tok'
+        # if the file doesn't exist, don't pass it to tooltool (it will just
+        # fail).  In taskcluster, this will work OK as the relengapi-proxy will
+        # take care of auth.  Everywhere else, we'll get auth failures if
+        # necessary.
+        if os.path.exists(fn):
+            return fn
 
     def tooltool_fetch(self, manifest, bootstrap_cmd=None,
                        output_dir=None, privileged=False, cache=None):
         """docstring for tooltool_fetch"""
         tooltool = self.query_exe('tooltool.py', return_type='list')
 
         if self.config.get("developer_mode"):
             tooltool = [bin for bin in tooltool if os.path.exists(bin)]
--- a/scripts/desktop_l10n.py
+++ b/scripts/desktop_l10n.py
@@ -1295,22 +1295,28 @@ class DesktopSingleLocale(LocalesMixin, 
                     pgc_files.append(os.path.join(dirpath, pgc))
         return pgc_files
 
     # TODO: replace with ToolToolMixin
     def _get_tooltool_auth_file(self):
         # set the default authentication file based on platform; this
         # corresponds to where puppet puts the token
         if 'tooltool_authentication_file' in self.config:
-            return self.config['tooltool_authentication_file']
+            fn = self.config['tooltool_authentication_file']
+        elif self._is_windows():
+            fn = r'c:\builds\relengapi.tok'
+        else:
+            fn = '/builds/relengapi.tok'
 
-        if self._is_windows():
-            return r'c:\builds\relengapi.tok'
-        else:
-            return '/builds/relengapi.tok'
+        # if the file doesn't exist, don't pass it to tooltool (it will just
+        # fail).  In taskcluster, this will work OK as the relengapi-proxy will
+        # take care of auth.  Everywhere else, we'll get auth failures if
+        # necessary.
+        if os.path.exists(fn):
+            return fn
 
     def _run_tooltool(self):
         config = self.config
         dirs = self.query_abs_dirs()
         if not config.get('tooltool_manifest_src'):
             return self.warning(ERROR_MSGS['tooltool_manifest_undetermined'])
         fetch_script_path = os.path.join(dirs['abs_tools_dir'],
                                          'scripts/tooltool/tooltool_wrapper.sh')
@@ -1319,17 +1325,19 @@ class DesktopSingleLocale(LocalesMixin, 
         cmd = [
             'sh',
             fetch_script_path,
             tooltool_manifest_path,
             config['tooltool_url'],
             config['tooltool_bootstrap'],
         ]
         cmd.extend(config['tooltool_script'])
-        cmd.extend(['--authentication-file', self._get_tooltool_auth_file()])
+        auth_file = self._get_tooltool_auth_file()
+        if auth_file and os.path.exists(auth_file):
+            cmd.extend(['--authentication-file', auth_file])
         self.info(str(cmd))
         self.run_command(cmd, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True)
 
     def _create_base_dirs(self):
         config = self.config
         dirs = self.query_abs_dirs()
         for make_dir in config.get('make_dirs', []):
             dirname = os.path.join(dirs['abs_objdir'], make_dir)