Bug 1527118: Ensure all tests are using tooltool caches r=aki
authorChris AtLee <catlee@mozilla.com>
Tue, 12 Feb 2019 16:48:35 +0000
changeset 458850 e7603feb2252
parent 458849 82a6589f4b91
child 458851 8c306186cbd7
push id35548
push useropoprus@mozilla.com
push dateWed, 13 Feb 2019 09:48:26 +0000
treeherdermozilla-central@93e37c529818 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1527118
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1527118: Ensure all tests are using tooltool caches r=aki respect TOOLTOOL_CACHE environment variable in mixin Differential Revision: https://phabricator.services.mozilla.com/D19444
taskcluster/taskgraph/transforms/job/mozharness_test.py
taskcluster/taskgraph/transforms/tests.py
testing/mozharness/mozharness/mozilla/tooltool.py
--- a/taskcluster/taskgraph/transforms/job/mozharness_test.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py
@@ -116,17 +116,18 @@ def mozharness_test_on_docker(config, jo
         env['MOZHARNESS_ACTIONS'] = ' '.join(mozharness['actions'])
 
     if config.params.is_try():
         env['TRY_COMMIT_MSG'] = config.params['message']
 
     # handle some of the mozharness-specific options
 
     if mozharness['tooltool-downloads']:
-        docker_worker_add_tooltool(config, job, taskdesc, internal=True)
+        internal = mozharness['tooltool-downloads'] == 'internal'
+        docker_worker_add_tooltool(config, job, taskdesc, internal=internal)
 
     if test['reboot']:
         raise Exception('reboot: {} not supported on generic-worker'.format(test['reboot']))
 
     # assemble the command line
     command = [
         '{workdir}/bin/run-task'.format(**run),
     ]
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -292,18 +292,23 @@ test_description_schema = Schema({
         Required('extra-options'): optionally_keyed_by(
             'test-platform',
             [basestring]),
 
         # the artifact name (including path) to test on the build task; this is
         # generally set in a per-kind transformation
         Optional('build-artifact-name'): basestring,
 
-        # If true, tooltool downloads will be enabled via relengAPIProxy.
-        Required('tooltool-downloads'): bool,
+        # If not false, tooltool downloads will be enabled via relengAPIProxy
+        # for either just public files, or all files.  Not supported on Windows
+        Required('tooltool-downloads'): Any(
+            False,
+            'public',
+            'internal',
+        ),
 
         # Add --blob-upload-branch=<project> mozharness parameter
         Optional('include-blob-upload-branch'): bool,
 
         # The setting for --download-symbols (if omitted, the option will not
         # be passed to mozharness)
         Optional('download-symbols'): Any(True, 'ondemand'),
 
@@ -422,17 +427,17 @@ def handle_keyed_by_mozharness(config, t
 
 
 @transforms.add
 def set_defaults(config, tests):
     for test in tests:
         build_platform = test['build-platform']
         if build_platform.startswith('android'):
             # all Android test tasks download internal objects from tooltool
-            test['mozharness']['tooltool-downloads'] = True
+            test['mozharness']['tooltool-downloads'] = 'internal'
             test['mozharness']['actions'] = ['get-secrets']
             # Fennec is non-e10s; geckoview handled in set_target
             test['e10s'] = False
             # loopback-video is always true for Android, but false for other
             # platform phyla
             test['loopback-video'] = True
         else:
             # all non-android tests want to run the bits that require node
@@ -468,17 +473,17 @@ def set_defaults(config, tests):
         test.setdefault('loopback-video', False)
         test.setdefault('docker-image', {'in-tree': 'desktop1604-test'})
         test.setdefault('checkout', False)
         test.setdefault('serviceworker-e10s', False)
         test.setdefault('require-signed-extensions', False)
 
         test['mozharness'].setdefault('extra-options', [])
         test['mozharness'].setdefault('requires-signed-builds', False)
-        test['mozharness'].setdefault('tooltool-downloads', False)
+        test['mozharness'].setdefault('tooltool-downloads', 'public')
         test['mozharness'].setdefault('set-moz-node-path', False)
         test['mozharness'].setdefault('chunked', False)
         test['mozharness'].setdefault('chunking-args', 'this-chunk')
         yield test
 
 
 transforms.add_validate(test_description_schema)
 
--- a/testing/mozharness/mozharness/mozilla/tooltool.py
+++ b/testing/mozharness/mozharness/mozilla/tooltool.py
@@ -42,16 +42,19 @@ class TooltoolMixin(object):
         # take care of auth.  Everywhere else, we'll get auth failures if
         # necessary.
         if os.path.exists(fn):
             return fn
 
     def tooltool_fetch(self, manifest,
                        output_dir=None, privileged=False, cache=None):
         """docstring for tooltool_fetch"""
+        if cache is None:
+            cache = os.environ.get('TOOLTOOL_CACHE')
+
         for d in (output_dir, cache):
             if d is not None and not os.path.exists(d):
                 self.mkdir_p(d)
         if self.topsrcdir:
             cmd = [
                 sys.executable, '-u',
                 os.path.join(self.topsrcdir, 'mach'),
                 'artifact',