Bug 1466537: Disable full crashsymbols by branch r=ted
authorChris AtLee <catlee@mozilla.com>
Mon, 30 Jul 2018 20:23:14 +0000
changeset 429311 76b7549059a4aaa3ce26587586be3e1ed3bbdbd1
parent 429310 985e470b78d88f0edd223118cfcbb9b8b5670b1a
child 429312 05ab9cc2b470ce9e445d37bc822de2f6d9c628fd
push id105875
push usershindli@mozilla.com
push dateTue, 31 Jul 2018 10:11:53 +0000
treeherdermozilla-inbound@b28bc7c7b6f8 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1466537
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1466537: Disable full crashsymbols by branch r=ted Differential Revision: https://phabricator.services.mozilla.com/D1558
Makefile.in
taskcluster/ci/build/linux.yml
taskcluster/ci/build/macosx.yml
taskcluster/ci/build/windows.yml
taskcluster/docs/attributes.rst
taskcluster/taskgraph/transforms/build.py
taskcluster/taskgraph/transforms/upload_symbols.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -273,17 +273,23 @@ symbolsfullarchive: prepsymbolsarchive
 
 .PHONY: symbolsarchive
 symbolsarchive: prepsymbolsarchive
 	$(RM) '$(DIST)/$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip'
 	$(call py_action,symbols_archive,'$(DIST)/$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip' \
                                      $(abspath $(DIST)/crashreporter-symbols))
 
 ifdef MOZ_CRASHREPORTER
+# Set MOZ_DISABLE_FULL_SYMBOLS to disable generation and upload of the full
+# crashreporter symbols archives
+ifdef MOZ_DISABLE_FULL_SYMBOLS
+buildsymbols: symbolsarchive
+else
 buildsymbols: symbolsfullarchive symbolsarchive
+endif # MOZ_DISABLE_FULL_SYMBOLS
 else
 buildsymbols:
 endif
 
 uploadsymbols:
 ifdef MOZ_CRASHREPORTER
 	$(PYTHON) -u $(topsrcdir)/toolkit/crashreporter/tools/upload_symbols.py '$(DIST)/$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip'
 endif
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -1,13 +1,15 @@
 linux64/opt:
     description: "Linux64 Opt"
     index:
         product: firefox
         job-name: linux64-opt
+    attributes:
+        enable-full-crashsymbols: true
     treeherder:
         platform: linux64/opt
         symbol: B
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 7200
     run:
         using: mozharness
@@ -86,16 +88,18 @@ linux64-dmd/opt:
         - linux64-rust-size
         - linux64-sccache
 
 linux64/pgo:
     description: "Linux64 PGO"
     index:
         product: firefox
         job-name: linux64-pgo
+    attributes:
+        enable-full-crashsymbols: true
     treeherder:
         platform: linux64/pgo
         symbol: B
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 7200
     run:
         using: mozharness
@@ -146,16 +150,18 @@ linux64-fuzzing/debug:
         - linux64-rust
         - linux64-rust-size
 
 linux64/debug:
     description: "Linux64 Debug"
     index:
         product: firefox
         job-name: linux64-debug
+    attributes:
+        enable-full-crashsymbols: true
     treeherder:
         platform: linux64/debug
         symbol: B
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 3600
     run:
         using: mozharness
@@ -202,16 +208,17 @@ linux64-plain/debug:
         - linux64-clang
         - linux64-gcc
         - linux64-rust
 
 linux64-devedition-nightly/opt:
     description: "Linux64 devedition Nightly"
     attributes:
         nightly: true
+        enable-full-crashsymbols: true
     shipping-phase: build
     shipping-product: devedition
     index:
         product: devedition
         job-name: linux64-opt
         type: nightly
     treeherder:
         platform: linux64-devedition/opt
@@ -298,16 +305,18 @@ linux64-base-toolchains/debug:
         - linux64-rust-1.27
         - linux64-sccache
 
 linux/opt:
     description: "Linux32 Opt"
     index:
         product: firefox
         job-name: linux-opt
+    attributes:
+        enable-full-crashsymbols: true
     treeherder:
         platform: linux32/opt
         symbol: B
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         docker-image: {in-tree: debian7-i386-build}
         max-run-time: 7200
     run:
@@ -327,16 +336,18 @@ linux/opt:
         - linux64-rust-size
         - linux64-sccache
 
 linux/debug:
     description: "Linux32 Debug"
     index:
         product: firefox
         job-name: linux-debug
+    attributes:
+        enable-full-crashsymbols: true
     treeherder:
         platform: linux32/debug
         symbol: B
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         docker-image: {in-tree: debian7-i386-build}
         max-run-time: 3600
     run:
@@ -357,16 +368,18 @@ linux/debug:
         - linux64-rust-size
         - linux64-sccache
 
 linux/pgo:
     description: "Linux32 PGO"
     index:
         product: firefox
         job-name: linux-pgo
+    attributes:
+        enable-full-crashsymbols: true
     treeherder:
         platform: linux32/pgo
         symbol: B
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         docker-image: {in-tree: debian7-i386-build}
         max-run-time: 7200
     run:
@@ -454,16 +467,17 @@ linux-rusttests/debug:
         - linux64-gcc
         - linux64-rust
         - linux64-sccache
 
 linux-devedition-nightly/opt:
     description: "Linux32 devedition Nightly"
     attributes:
         nightly: true
+        enable-full-crashsymbols: true
     shipping-phase: build
     shipping-product: devedition
     index:
         product: devedition
         job-name: linux-opt
         type: nightly
     treeherder:
         platform: linux32-devedition/opt
@@ -491,16 +505,17 @@ linux-devedition-nightly/opt:
         - linux64-rust
         - linux64-rust-size
         - linux64-sccache
 
 linux-nightly/opt:
     description: "Linux32 Nightly"
     attributes:
         nightly: true
+        enable-full-crashsymbols: true
     shipping-phase: build
     shipping-product: firefox
     index:
         product: firefox
         job-name: linux-opt
         type: nightly
     treeherder:
         platform: linux32/opt
@@ -522,17 +537,16 @@ linux-nightly/opt:
         need-xvfb: true
     toolchains:
         - linux64-clang
         - linux64-gcc
         - linux64-rust
         - linux64-rust-size
         - linux64-sccache
 
-
 linux64-asan/opt:
     description: "Linux64 Opt ASAN"
     index:
         product: firefox
         job-name: linux64-asan-opt
     treeherder:
         platform: linux64/asan
         symbol: Bo
@@ -751,16 +765,17 @@ linux64-lto/debug:
         - linux64-rust
         - linux64-rust-size
         - linux64-sccache
 
 linux64-nightly/opt:
     description: "Linux64 Nightly"
     attributes:
         nightly: true
+        enable-full-crashsymbols: true
     shipping-phase: build
     shipping-product: firefox
     index:
         product: firefox
         job-name: linux64-opt
         type: nightly
     treeherder:
         platform: linux64/opt
--- a/taskcluster/ci/build/macosx.yml
+++ b/taskcluster/ci/build/macosx.yml
@@ -1,13 +1,15 @@
 macosx64/debug:
     description: "MacOS X x64 Cross-compile"
     index:
         product: firefox
         job-name: macosx64-debug
+    attributes:
+        enable-full-crashsymbols: true
     treeherder:
         platform: osx-cross/debug
         symbol: B
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 3600
         env:
@@ -32,16 +34,18 @@ macosx64/debug:
         - linux64-rust-size
         - linux64-sccache
 
 macosx64/opt:
     description: "MacOS X x64 Cross-compile"
     index:
         product: firefox
         job-name: macosx64-opt
+    attributes:
+        enable-full-crashsymbols: true
     treeherder:
         platform: osx-cross/opt
         symbol: B
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 3600
         env:
@@ -134,16 +138,17 @@ macosx64-dmd/opt:
         - linux64-rust-macos
         - linux64-rust-size
         - linux64-sccache
 
 macosx64-devedition-nightly/opt:
     description: "MacOS X Dev Edition x64 Nightly"
     attributes:
         nightly: true
+        enable-full-crashsymbols: true
     shipping-phase: build
     shipping-product: devedition
     index:
         product: devedition
         job-name: macosx64-opt
         type: nightly
     treeherder:
         platform: osx-cross-devedition/opt
@@ -246,16 +251,17 @@ macosx64-add-on-devel/opt:
         - linux64-rust-macos
         - linux64-rust-size
         - linux64-sccache
 
 macosx64-nightly/opt:
     description: "MacOS X x64 Cross-compile Nightly"
     attributes:
         nightly: true
+        enable-full-crashsymbols: true
     shipping-phase: build
     shipping-product: firefox
     index:
         product: firefox
         job-name: macosx64-opt
         type: nightly
     treeherder:
         platform: osx-cross/opt
--- a/taskcluster/ci/build/windows.yml
+++ b/taskcluster/ci/build/windows.yml
@@ -1,13 +1,15 @@
 win32/debug:
     description: "Win32 Debug"
     index:
         product: firefox
         job-name: win32-debug
+    attributes:
+        enable-full-crashsymbols: true
     treeherder:
         platform: windows2012-32/debug
         symbol: B
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
     worker:
         max-run-time: 7200
         env:
@@ -27,16 +29,18 @@ win32/debug:
         - win64-rust-size
         - win64-sccache
 
 win32/opt:
     description: "Win32 Opt"
     index:
         product: firefox
         job-name: win32-opt
+    attributes:
+        enable-full-crashsymbols: true
     treeherder:
         platform: windows2012-32/opt
         symbol: B
         tier: 1
     stub-installer:
         # Beta and release use a stub-requiring update channel
         by-project:
             default: false
@@ -94,16 +98,18 @@ win32-dmd/opt:
         - win64-rust-size
         - win64-sccache
 
 win32/pgo:
     description: "Win32 Opt PGO"
     index:
         product: firefox
         job-name: win32-pgo
+    attributes:
+        enable-full-crashsymbols: true
     treeherder:
         platform: windows2012-32/pgo
         symbol: B
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
     worker:
         max-run-time: 9000
         env:
@@ -123,16 +129,18 @@ win32/pgo:
         - win64-rust-size
         - win64-sccache
 
 win64/debug:
     description: "Win64 Debug"
     index:
         product: firefox
         job-name: win64-debug
+    attributes:
+        enable-full-crashsymbols: true
     treeherder:
         platform: windows2012-64/debug
         symbol: B
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
     worker:
         max-run-time: 7200
         env:
@@ -181,16 +189,18 @@ win64-plain/debug:
         - win64-clang-cl
         - win64-rust
 
 win64/opt:
     description: "Win64 Opt"
     index:
         product: firefox
         job-name: win64-opt
+    attributes:
+        enable-full-crashsymbols: true
     treeherder:
         platform: windows2012-64/opt
         symbol: B
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
     worker:
         max-run-time: 7200
         env:
@@ -274,16 +284,17 @@ win64-dmd/opt:
 win32-nightly/opt:
     description: "Win32 Nightly"
     index:
         product: firefox
         job-name: win32-opt
         type: nightly
     attributes:
         nightly: true
+        enable-full-crashsymbols: true
     stub-installer:
         by-project:
             default: false
             mozilla-central: true
             try: true
             mozilla-beta: true
             mozilla-release: true
     shipping-phase: build
@@ -317,16 +328,17 @@ win32-nightly/opt:
 win64-nightly/opt:
     description: "Win64 Nightly"
     index:
         product: firefox
         job-name: win64-opt
         type: nightly
     attributes:
         nightly: true
+        enable-full-crashsymbols: true
     shipping-phase: build
     shipping-product: firefox
     treeherder:
         platform: windows2012-64/opt
         symbol: N
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
     worker:
@@ -350,16 +362,18 @@ win64-nightly/opt:
         - win64-rust-size
         - win64-sccache
 
 win64/pgo:
     description: "Win64 Opt PGO"
     index:
         product: firefox
         job-name: win64-pgo
+    attributes:
+        enable-full-crashsymbols: true
     treeherder:
         platform: windows2012-64/pgo
         symbol: B
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
     worker:
         max-run-time: 10800
         env:
@@ -688,16 +702,17 @@ win64-asan-reporter-nightly/opt:
 win32-devedition-nightly/opt:
     description: "Win32 Dev Edition Nightly"
     index:
         product: devedition
         job-name: win32-opt
         type: nightly
     attributes:
         nightly: true
+        enable-full-crashsymbols: true
     stub-installer:
         by-project:
             default: false
             mozilla-central: true
             try: true
             mozilla-beta: true
             mozilla-release: true
     shipping-phase: build
@@ -732,16 +747,17 @@ win32-devedition-nightly/opt:
 win64-devedition-nightly/opt:
     description: "Win64 Dev Edition Nightly"
     index:
         product: devedition
         job-name: win64-opt
         type: nightly
     attributes:
         nightly: true
+        enable-full-crashsymbols: true
     shipping-phase: build
     shipping-product: devedition
     treeherder:
         platform: windows2012-64-devedition/opt
         symbol: N
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
     worker:
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -220,8 +220,15 @@ build the head. If we don't point at the
 (build, promote, push).
 
 artifact_prefix
 ===============
 Most taskcluster artifacts are public, so we've hardcoded ``public/build`` in a
 lot of places. To support private artifacts, we've moved this to the
 ``artifact_prefix`` attribute. It will default to ``public/build`` but will be
 overrideable per-task.
+
+enable-full-crashsymbols
+========================
+In automation, full crashsymbol package generation is normally disabled.  For
+build kinds where the full crashsymbols should be enabled, set this attribute
+to True. The full symbol packages will then be generated and uploaded on
+release branches and on try.
--- a/taskcluster/taskgraph/transforms/build.py
+++ b/taskcluster/taskgraph/transforms/build.py
@@ -4,19 +4,23 @@
 """
 Apply some defaults and minor modifications to the jobs defined in the build
 kind.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import RELEASE_PROJECTS
 from taskgraph.util.schema import resolve_keyed_by
 from taskgraph.util.workertypes import worker_type_implementation
 
+import logging
+logger = logging.getLogger(__name__)
+
 transforms = TransformSequence()
 
 
 @transforms.add
 def set_defaults(config, jobs):
     """Set defaults, including those that differ per worker implementation"""
     for job in jobs:
         job['treeherder'].setdefault('kind', 'build')
@@ -53,8 +57,24 @@ def set_env(config, jobs):
     """Set extra environment variables from try command line."""
     env = []
     if config.params['try_mode'] == 'try_option_syntax':
         env = config.params['try_options']['env'] or []
     for job in jobs:
         if env:
             job['worker']['env'].update(dict(x.split('=') for x in env))
         yield job
+
+
+@transforms.add
+def enable_full_crashsymbols(config, jobs):
+    """Enable full crashsymbols on jobs with
+    'enable-full-crashsymbols' set to True and on release branches, or
+    on try"""
+    branches = RELEASE_PROJECTS | {'try', }
+    for job in jobs:
+        enable_full_crashsymbols = job['attributes'].get('enable-full-crashsymbols')
+        if enable_full_crashsymbols and config.params['project'] in branches:
+            logger.debug("Enabling full symbol generation for %s", job['name'])
+        else:
+            logger.debug("Disabling full symbol generation for %s", job['name'])
+            job['worker']['env']['MOZ_DISABLE_FULL_SYMBOLS'] = '1'
+        yield job
--- a/taskcluster/taskgraph/transforms/upload_symbols.py
+++ b/taskcluster/taskgraph/transforms/upload_symbols.py
@@ -5,27 +5,35 @@
 Transform the upload-symbols task description template,
 taskcluster/ci/upload-symbols/job-template.yml into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 
+import logging
+logger = logging.getLogger(__name__)
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def fill_template(config, tasks):
     for task in tasks:
         dep = task['dependent-task']
 
         # Fill out the dynamic fields in the task description
         task['label'] = dep.label + '-upload-symbols'
+
+        # Skip tasks where we don't have the full crashsymbols enabled
+        if not dep.attributes.get('enable-full-crashsymbols'):
+            logger.debug("Skipping upload symbols task for %s", task['label'])
+            continue
+
         task['dependencies'] = {'build': dep.label}
         task['worker']['env']['GECKO_HEAD_REPOSITORY'] = config.params['head_repository']
         task['worker']['env']['GECKO_HEAD_REV'] = config.params['head_rev']
         task['worker']['env']['SYMBOL_SECRET'] = task['worker']['env']['SYMBOL_SECRET'].format(
             level=config.params['level'])
 
         build_platform = dep.attributes.get('build_platform')
         build_type = dep.attributes.get('build_type')