Merge mozilla-central to autoland. CLOSED TREE
authorDorel Luca <dluca@mozilla.com>
Sun, 10 Nov 2019 05:05:40 +0200
changeset 501411 110e47c61308a4691e521cf59a6a05a65815d870
parent 501410 d967ac6c56aeec99c2fba3814c8550a1c5971384 (current diff)
parent 501365 33f64c1ef3e40dc619c5eb8e43c5967959f57b29 (diff)
child 501412 cdfe04574b0c6afba17637ee8e7419c3f2d6068f
push id36791
push usercsabou@mozilla.com
push dateSun, 10 Nov 2019 09:53:30 +0000
treeherdermozilla-central@72c52c0101cf [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
milestone72.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Merge mozilla-central to autoland. CLOSED TREE
--- a/taskcluster/ci/build/windows.yml
+++ b/taskcluster/ci/build/windows.yml
@@ -1,16 +1,20 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 job-defaults:
     run:
         using: mozharness
         use-caches: false
+        tooltool-downloads: internal
+        extra-config:
+            "tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/']
+            'tooltool_url': 'http://taskcluster/tooltool.mozilla-releng.net/'
 
 win32/debug:
     description: "Win32 Debug"
     index:
         product: firefox
         job-name: win32-debug
     attributes:
         enable-full-crashsymbols: true
--- a/taskcluster/ci/instrumented-build/kind.yml
+++ b/taskcluster/ci/instrumented-build/kind.yml
@@ -141,18 +141,21 @@ jobs:
         run:
             use-caches: false
             options: [append-env-variables-from-configs]
             mozconfig-variant: profile-generate
             config:
                 - builds/releng_base_firefox.py
                 - builds/taskcluster_base_windows.py
                 - builds/taskcluster_base_win32.py
+            tooltool-downloads: internal
             extra-config:
                 stage_platform: win32
+                "tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/']
+                'tooltool_url': 'http://taskcluster/tooltool.mozilla-releng.net/'
         fetches:
             toolchain:
                 - win64-clang-cl
                 - win64-rust
                 - win64-rust-size
                 - win64-cbindgen
                 - win64-nasm
                 - win64-node
@@ -172,18 +175,21 @@ jobs:
         run:
             use-caches: false
             options: [append-env-variables-from-configs]
             mozconfig-variant: profile-generate
             config:
                 - builds/releng_base_firefox.py
                 - builds/taskcluster_base_windows.py
                 - builds/taskcluster_base_win64.py
+            tooltool-downloads: internal
             extra-config:
                 stage_platform: win64
+                "tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/']
+                'tooltool_url': 'http://taskcluster/tooltool.mozilla-releng.net/'
         fetches:
             toolchain:
                 - win64-clang-cl
                 - win64-rust
                 - win64-rust-size
                 - win64-cbindgen
                 - win64-nasm
                 - win64-node
--- a/taskcluster/ci/searchfox/kind.yml
+++ b/taskcluster/ci/searchfox/kind.yml
@@ -114,16 +114,20 @@ jobs:
             use-caches: false
             options: [append-env-variables-from-configs]
             script: mozharness/scripts/fx_desktop_build.py
             config:
                 - builds/releng_base_firefox.py
                 - builds/taskcluster_base_windows.py
                 - builds/taskcluster_base_win64.py
                 - builds/taskcluster_sub_win64/searchfox_debug.py
+            tooltool-downloads: internal
+            extra-config:
+                "tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/']
+                'tooltool_url': 'http://taskcluster/tooltool.mozilla-releng.net/'
         fetches:
             toolchain:
                 - win64-clang-cl
                 - win64-node
                 - win64-rust
                 - win64-cbindgen
                 - win64-nasm
 
--- a/taskcluster/ci/static-analysis-autotest/kind.yml
+++ b/taskcluster/ci/static-analysis-autotest/kind.yml
@@ -76,18 +76,21 @@ jobs:
             env:
                 TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
         run:
             config:
                 - builds/releng_base_firefox.py
                 - builds/taskcluster_base_windows.py
                 - builds/taskcluster_base_win64.py
                 - builds/taskcluster_sub_win64/debug.py
+            tooltool-downloads: internal
             extra-config:
                 mozconfig_variant: debug
+                "tooltool_servers": ['http://taskcluster/tooltool.mozilla-releng.net/']
+                'tooltool_url': 'http://taskcluster/tooltool.mozilla-releng.net/'
         fetches:
             toolchain:
                 - win64-clang-cl
                 - win64-rust
                 - win64-sccache
                 - win64-cbindgen
                 - win64-clang-tidy
                 - win64-nasm
--- a/taskcluster/ci/toolchain/cbindgen.yml
+++ b/taskcluster/ci/toolchain/cbindgen.yml
@@ -55,11 +55,12 @@ win64-cbindgen:
     worker:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
     run:
         arguments: ['x86_64-pc-windows-msvc']
         resources:
             - taskcluster/scripts/misc/tooltool-download.sh
         toolchain-artifact: public/build/cbindgen.tar.bz2
+        tooltool-downloads: internal
     fetches:
         toolchain:
             - win64-rust-1.34
--- a/taskcluster/ci/toolchain/cctools-port.yml
+++ b/taskcluster/ci/toolchain/cctools-port.yml
@@ -13,12 +13,13 @@ job-defaults:
             - libtapi
 
 linux64-cctools-port:
     treeherder:
         symbol: TL(cctools)
     run:
         script: build-cctools-port.sh
         toolchain-artifact: public/build/cctools.tar.xz
+        tooltool-downloads: internal
     fetches:
         toolchain:
             - linux64-clang-9
             - linux64-binutils
--- a/taskcluster/ci/toolchain/clang-tidy.yml
+++ b/taskcluster/ci/toolchain/clang-tidy.yml
@@ -82,12 +82,13 @@ win64-clang-tidy:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
     run:
         script: build-clang.sh
         arguments:
             - 'build/build-clang/clang-tidy-win64.json'
         resources:
             - 'build/build-clang/clang-tidy-win64.json'
         toolchain-artifact: public/build/clang-tidy.tar.bz2
+        tooltool-downloads: internal
     fetches:
         fetch:
             - cmake
             - ninja
--- a/taskcluster/ci/toolchain/clang.yml
+++ b/taskcluster/ci/toolchain/clang.yml
@@ -245,13 +245,14 @@ win64-clang-cl:
     run:
         script: build-clang.sh
         arguments:
             - 'build/build-clang/clang-win64.json'
         resources:
             - 'build/build-clang/clang-win64.json'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/clang.tar.bz2
+        tooltool-downloads: internal
     fetches:
         fetch:
             - clang-9
             - cmake
             - ninja
--- a/taskcluster/ci/toolchain/gn.yml
+++ b/taskcluster/ci/toolchain/gn.yml
@@ -51,11 +51,12 @@ win32-gn:
         max-run-time: 3600
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
     run:
         script: build-gn-win32.sh
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/gn.tar.bz2
+        tooltool-downloads: internal
     fetches:
         fetch:
             - ninja
--- a/taskcluster/ci/toolchain/grcov.yml
+++ b/taskcluster/ci/toolchain/grcov.yml
@@ -37,11 +37,12 @@ win64-grcov:
     worker-type: b-win2012
     worker:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
     run:
         toolchain-artifact: public/build/grcov.tar.bz2
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        tooltool-downloads: internal
     fetches:
         toolchain:
             - win64-rust-1.34
--- a/taskcluster/ci/toolchain/rust-size.yml
+++ b/taskcluster/ci/toolchain/rust-size.yml
@@ -28,11 +28,12 @@ win64-rust-size:
     worker-type: b-win2012
     worker:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
     run:
         toolchain-artifact: public/build/rust-size.tar.bz2
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        tooltool-downloads: internal
     fetches:
         toolchain:
             - win64-rust-1.28
--- a/taskcluster/ci/toolchain/sccache.yml
+++ b/taskcluster/ci/toolchain/sccache.yml
@@ -60,11 +60,12 @@ win64-sccache:
     worker:
         max-run-time: 3600
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
     run:
         toolchain-artifact: public/build/sccache.tar.bz2
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        tooltool-downloads: internal
     fetches:
         toolchain:
             - win64-rust-1.34
--- a/taskcluster/ci/webrender/kind.yml
+++ b/taskcluster/ci/webrender/kind.yml
@@ -226,16 +226,17 @@ jobs:
         fetches:
             toolchain:
                 - win64-rust
                 - wrench-deps
         run:
             using: run-task
             use-caches: false
             command: '$GECKO_PATH/taskcluster/scripts/misc/wrench-windows-tests.sh'
+            tooltool-downloads: internal
         treeherder:
             platform: windows10-64-qr/release
             symbol: WR(wrench)
         when:
             files-changed:
                 - 'build/win64/mozconfig.vs2017'
                 - 'browser/config/tooltool-manifests/win64/webrender.manifest'
                 - 'gfx/wr/**'
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -205,37 +205,38 @@ def setup_secrets(config, job, taskdesc)
     secrets = job['run']['secrets']
     if secrets is True:
         secrets = ['*']
     for sec in secrets:
         taskdesc['scopes'].append(SECRET_SCOPE.format(
             job['treeherder']['kind'], config.params['level'], sec))
 
 
-def docker_worker_add_tooltool(config, job, taskdesc, internal=False):
+def add_tooltool(config, job, taskdesc, internal=False):
     """Give the task access to tooltool.
 
     Enables the tooltool cache. Adds releng proxy. Configures scopes.
 
     By default, only public tooltool access will be granted. Access to internal
     tooltool can be enabled via ``internal=True``.
 
     This can only be used with ``run-task`` tasks, as the cache name is
     reserved for use with ``run-task``.
     """
 
-    assert job['worker']['implementation'] in ('docker-worker',)
+    if job['worker']['implementation'] in ('docker-worker',):
+        level = config.params['level']
+        add_cache(job, taskdesc, 'tooltool-cache'.format(level),
+                  '{workdir}/tooltool-cache'.format(**job['run']))
 
-    level = config.params['level']
-    add_cache(job, taskdesc, 'tooltool-cache'.format(level),
-              '{workdir}/tooltool-cache'.format(**job['run']))
-
-    taskdesc['worker'].setdefault('env', {}).update({
-        'TOOLTOOL_CACHE': '{workdir}/tooltool-cache'.format(**job['run']),
-    })
+        taskdesc['worker'].setdefault('env', {}).update({
+            'TOOLTOOL_CACHE': '{workdir}/tooltool-cache'.format(**job['run']),
+        })
+    elif not internal:
+        return
 
     taskdesc['worker']['taskcluster-proxy'] = True
     taskdesc['scopes'].extend([
         'project:releng:services/tooltool/api/download/public',
     ])
 
     if internal:
         taskdesc['scopes'].extend([
--- a/taskcluster/taskgraph/transforms/job/hazard.py
+++ b/taskcluster/taskgraph/transforms/job/hazard.py
@@ -13,17 +13,17 @@ from voluptuous import Required, Optiona
 from taskgraph.transforms.job import (
     run_job_using,
     configure_taskdesc_for_run,
 )
 from taskgraph.transforms.job.common import (
     docker_worker_add_workspace_cache,
     setup_secrets,
     docker_worker_add_artifacts,
-    docker_worker_add_tooltool,
+    add_tooltool,
 )
 
 haz_run_schema = Schema({
     Required('using'): 'hazard',
 
     # The command to run within the task image (passed through to the worker)
     Required('command'): basestring,
 
@@ -46,17 +46,17 @@ haz_run_schema = Schema({
 def docker_worker_hazard(config, job, taskdesc):
     run = job['run']
 
     worker = taskdesc['worker'] = job['worker']
     worker['artifacts'] = []
 
     docker_worker_add_artifacts(config, job, taskdesc)
     docker_worker_add_workspace_cache(config, job, taskdesc)
-    docker_worker_add_tooltool(config, job, taskdesc)
+    add_tooltool(config, job, taskdesc)
     setup_secrets(config, job, taskdesc)
 
     env = worker['env']
     env.update({
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
         'MOZ_SCM_LEVEL': config.params['level'],
     })
 
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -236,27 +236,29 @@ def mozharness_on_generic_worker(config,
     assert (
         job["worker"]["os"] == "windows"
     ), "only supports windows right now: {}".format(job["label"])
 
     run = job['run']
 
     # fail if invalid run options are included
     invalid = []
-    for prop in ['tooltool-downloads', 'taskcluster-proxy', 'need-xvfb']:
+    for prop in ['need-xvfb']:
         if prop in run and run.pop(prop):
             invalid.append(prop)
     if not run.pop('keep-artifacts', True):
         invalid.append('keep-artifacts')
     if invalid:
         raise Exception("Jobs run using mozharness on Windows do not support properties " +
                         ', '.join(invalid))
 
     worker = taskdesc['worker'] = job['worker']
 
+    worker['taskcluster-proxy'] = run.pop('taskcluster-proxy', None)
+
     setup_secrets(config, job, taskdesc)
 
     taskdesc['worker'].setdefault('artifacts', []).append({
         'name': 'public/logs',
         'path': 'logs',
         'type': 'directory'
     })
     if not worker.get('skip-artifacts', False):
--- a/taskcluster/taskgraph/transforms/job/mozharness_test.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py
@@ -388,16 +388,19 @@ def mozharness_test_on_generic_worker(co
         'workdir': run['workdir'],
         'tooltool-downloads': mozharness['tooltool-downloads'],
         'checkout': test['checkout'],
         'command': mh_command,
         'using': 'run-task',
     }
     if is_bitbar:
         job['run']['run-as-root'] = True
+        # FIXME: The bitbar config incorrectly requests internal tooltool downloads
+        # so force it off here.
+        job['run']['tooltool-downloads'] = False
     configure_taskdesc_for_run(config, job, taskdesc, worker['implementation'])
 
 
 @run_job_using('script-engine-autophone', 'mozharness-test', schema=mozharness_test_run_schema)
 def mozharness_test_on_script_engine_autophone(config, job, taskdesc):
     test = taskdesc['run']['test']
     mozharness = test['mozharness']
     worker = taskdesc['worker']
--- a/taskcluster/taskgraph/transforms/job/run_task.py
+++ b/taskcluster/taskgraph/transforms/job/run_task.py
@@ -10,17 +10,17 @@ from __future__ import absolute_import, 
 from six import text_type
 
 from mozpack import path
 
 from taskgraph.transforms.task import taskref_or_string
 from taskgraph.transforms.job import run_job_using
 from taskgraph.util.schema import Schema
 from taskgraph.transforms.job.common import (
-        docker_worker_add_tooltool,
+        add_tooltool,
         support_vcs_checkout
 )
 from voluptuous import Any, Optional, Required
 
 run_task_schema = Schema({
     Required('using'): 'run-task',
 
     # if true, add a cache at ~worker/.cache, which is where things like pip
@@ -104,17 +104,17 @@ def script_url(config, script):
 def docker_worker_run_task(config, job, taskdesc):
     run = job['run']
     worker = taskdesc['worker'] = job['worker']
     command = ['/builds/worker/bin/run-task']
     common_setup(config, job, taskdesc, command)
 
     if run['tooltool-downloads']:
         internal = run['tooltool-downloads'] == 'internal'
-        docker_worker_add_tooltool(config, job, taskdesc, internal=internal)
+        add_tooltool(config, job, taskdesc, internal=internal)
 
     if run.get('cache-dotcache'):
         worker['caches'].append({
             'type': 'persistent',
             'name': '{project}-dotcache'.format(**config.params),
             'mount-point': '{workdir}/.cache'.format(**run),
             'skip-untrusted': True,
         })
@@ -150,16 +150,20 @@ def docker_worker_run_task(config, job, 
 @run_job_using("generic-worker", "run-task", schema=run_task_schema, defaults=worker_defaults)
 def generic_worker_run_task(config, job, taskdesc):
     run = job['run']
     worker = taskdesc['worker'] = job['worker']
     is_win = worker['os'] == 'windows'
     is_mac = worker['os'] == 'macosx'
     is_bitbar = worker['os'] == 'linux-bitbar'
 
+    if run['tooltool-downloads']:
+        internal = run['tooltool-downloads'] == 'internal'
+        add_tooltool(config, job, taskdesc, internal=internal)
+
     if is_win:
         command = ['C:/mozilla-build/python3/python3.exe', 'run-task']
     elif is_mac:
         command = ['/tools/python37/bin/python3.7', 'run-task']
         if job['worker-type'].endswith('1014'):
             command = ['/usr/local/bin/python3', 'run-task']
     else:
         command = ['./run-task']
--- a/taskcluster/taskgraph/transforms/job/spidermonkey.py
+++ b/taskcluster/taskgraph/transforms/job/spidermonkey.py
@@ -12,17 +12,17 @@ from voluptuous import Required, Any, Op
 
 from taskgraph.transforms.job import (
     run_job_using,
     configure_taskdesc_for_run,
 )
 from taskgraph.transforms.job.common import (
     docker_worker_add_artifacts,
     generic_worker_add_artifacts,
-    docker_worker_add_tooltool,
+    add_tooltool,
 )
 
 sm_run_schema = Schema({
     Required('using'): Any('spidermonkey', 'spidermonkey-package', 'spidermonkey-mozjs-crate',
                            'spidermonkey-rust-bindings'),
 
     # SPIDERMONKEY_VARIANT and SPIDERMONKEY_PLATFORM
     Required('spidermonkey-variant'): basestring,
@@ -47,17 +47,17 @@ def docker_worker_spidermonkey(config, j
     worker.setdefault('caches', []).append({
         'type': 'persistent',
         'name': '{}-build-spidermonkey-workspace'.format(config.params['project']),
         'mount-point': "{workdir}/workspace".format(**run),
         'skip-untrusted': True,
     })
 
     docker_worker_add_artifacts(config, job, taskdesc)
-    docker_worker_add_tooltool(config, job, taskdesc)
+    add_tooltool(config, job, taskdesc)
 
     env = worker.setdefault('env', {})
     env.update({
         'MOZHARNESS_DISABLE': 'true',
         'SPIDERMONKEY_VARIANT': run.pop('spidermonkey-variant'),
         'MOZ_BUILD_DATE': config.params['moz_build_date'],
         'MOZ_SCM_LEVEL': config.params['level'],
         'GECKO_PATH': '{}/workspace/build/src'.format(run['workdir'])
--- a/taskcluster/taskgraph/util/taskcluster.py
+++ b/taskcluster/taskgraph/util/taskcluster.py
@@ -19,17 +19,17 @@ from taskgraph.util import yaml
 
 logger = logging.getLogger(__name__)
 
 # this is set to true for `mach taskgraph action-callback --test`
 testing = False
 
 # Default rootUrl to use if none is given in the environment; this should point
 # to the production Taskcluster deployment used for CI.
-PRODUCTION_TASKCLUSTER_ROOT_URL = 'https://taskcluster.net'
+PRODUCTION_TASKCLUSTER_ROOT_URL = 'https://firefox-ci-tc.services.mozilla.com'
 
 # the maximum number of parallel Taskcluster API calls to make
 CONCURRENCY = 50
 
 
 @memoize
 def get_root_url(use_proxy):
     """Get the current TASKCLUSTER_ROOT_URL.  When running in a task, this must
--- a/testing/raptor/raptor/results.py
+++ b/testing/raptor/raptor/results.py
@@ -421,18 +421,21 @@ class BrowsertimeResultsHandler(Perftest
                     )
 
             results.append(bt_result)
 
         return results
 
     def _extract_vmetrics_jobs(self, test, browsertime_json, browsertime_results):
         # XXX will do better later
-        url = ("https://queue.taskcluster.net/v1/task/%s/runs/0/artifacts/public/"
-               "test_info/" % os.environ.get("TASK_ID", "??"))
+        url = ("{root_url}/api/queue/v1/task/{task_id}/runs/0/artifacts/public/"
+               "test_info/".format(
+                   root_url=os.environ.get('TASKCLUSTER_ROOT_URL', 'taskcluster-root-url.invalid'),
+                   task_id=os.environ.get("TASK_ID", "??"),
+               ))
 
         json_url = url + "/".join(browsertime_json.split(os.path.sep)[-3:])
         files = []
         for res in browsertime_results:
             files.extend(res.get("files", {}).get("video", []))
         if len(files) == 0:
             # no video files.
             return None