Bug 1588538 - Use the new Windows dump_syms tool to dump symbols in local builds r=glandium
authorGabriele Svelto <gsvelto@mozilla.com>
Thu, 16 Jan 2020 14:14:13 +0000
changeset 510470 893e9fda4bf3be7283c8826950bad9b4c26abbcd
parent 510469 a51e1a67ed2e81713ca1fde450f9c754929137c6
child 510471 325eec16f032c255567b1081f941735677d9c5b2
push id37023
push userncsoregi@mozilla.com
push dateThu, 16 Jan 2020 21:45:49 +0000
treeherdermozilla-central@3f72a81bd12c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1588538
milestone74.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1588538 - Use the new Windows dump_syms tool to dump symbols in local builds r=glandium Differential Revision: https://phabricator.services.mozilla.com/D57094
browser/config/mozconfigs/win64/plain-opt
build/moz.configure/toolchain.configure
build/mozconfig.no-compile
build/mozconfig.win-common
python/mozboot/mozboot/base.py
python/mozboot/mozboot/bootstrap.py
python/mozboot/mozboot/mozillabuild.py
python/mozbuild/mozbuild/action/dumpsymbols.py
toolkit/crashreporter/google-breakpad/src/tools/windows/dump_syms/moz.build
toolkit/crashreporter/moz.build
toolkit/crashreporter/tools/unit-symbolstore.py
toolkit/moz.configure
tools/crashreporter/app.mozbuild
--- a/browser/config/mozconfigs/win64/plain-opt
+++ b/browser/config/mozconfigs/win64/plain-opt
@@ -1,11 +1,12 @@
 TOOLTOOL_DIR=${topsrcdir}
 export MAKECAB=$TOOLTOOL_DIR/makecab.exe
 export LLVM_CONFIG="${MOZ_FETCHES_DIR}/clang/bin/llvm-config"
+export DUMP_SYMS="${MOZ_FETCHES_DIR}/dump_syms/dump_syms.exe"
 
 RUSTC="${MOZ_FETCHES_DIR}/rustc/bin/rustc"
 CARGO="${MOZ_FETCHES_DIR}/rustc/bin/cargo"
 RUSTDOC="${MOZ_FETCHES_DIR}/rustc/bin/rustdoc"
 RUSTFMT="${MOZ_FETCHES_DIR}/rustc/bin/rustfmt"
 CBINDGEN="${MOZ_FETCHES_DIR}/cbindgen/cbindgen"
 
 NASM="${MOZ_FETCHES_DIR}/nasm/nasm.exe"
--- a/build/moz.configure/toolchain.configure
+++ b/build/moz.configure/toolchain.configure
@@ -709,16 +709,17 @@ def toolchain_search_path_for(host_or_ta
             result.append(vc_compiler_path[0])
 
         # Also add in the location to which `mach bootstrap` or
         # `mach artifact toolchain` installs clang, cbindgen, etc.
         bootstrapped = [
             os.path.join(mozbuild_state_path, *rest) for rest in (
                 ['clang', 'bin'],
                 ['cbindgen'],
+                ['dump_syms'],
                 ['nasm'],
                 ['lucetc'],
             )]
 
         # Also add the rustup install directory for cargo/rustc.
         rustup_path = os.path.expanduser(os.path.join('~', '.cargo', 'bin'))
         result.append(rustup_path)
 
--- a/build/mozconfig.no-compile
+++ b/build/mozconfig.no-compile
@@ -14,16 +14,17 @@ unset HOST_CXX
 unset LINKER
 unset RUSTC
 unset RUSTFLAGS
 unset CARGO
 unset RUSTDOC
 unset RUSTFMT
 unset CBINDGEN
 unset MAKECAB
+unset DUMP_SYMS
 unset TOOLCHAIN_PREFIX
 unset BINDGEN_CFLAGS
 unset LLVM_CONFIG
 unset LLVM_PROFDATA
 unset ENABLE_CLANG_PLUGIN
 unset MACOS_SDK_DIR
 unset MACOS_PRIVATE_FRAMEWORKS_DIR
 unset DIA_SDK_PATH
--- a/build/mozconfig.win-common
+++ b/build/mozconfig.win-common
@@ -1,14 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 TOOLTOOL_DIR=${TOOLTOOL_DIR:-$topsrcdir}
 export MAKECAB=$TOOLTOOL_DIR/makecab.exe
+export DUMP_SYMS="${MOZ_FETCHES_DIR}/dump_syms/dump_syms.exe"
 
 if [ -z "$USE_ARTIFACT" ]; then
     if [ -n "$TASKCLUSTER_PGO_PROFILE_USE" ]; then
         export MOZ_LTO=cross
         ac_add_options --enable-profile-use=cross
         ac_add_options --with-pgo-jarlog="${WORKSPACE}/fetches/en-US.log"
         ac_add_options --with-pgo-profile-path="${WORKSPACE}/fetches/merged.profdata"
     fi
--- a/python/mozboot/mozboot/base.py
+++ b/python/mozboot/mozboot/base.py
@@ -307,16 +307,22 @@ class BaseBootstrapper(object):
 
     def ensure_node_packages(self, state_dir, checkout_root):
         '''
         Install any necessary packages needed to supply NodeJS'''
         raise NotImplementedError(
             '%s does not yet implement ensure_node_packages()'
             % __name__)
 
+    def ensure_dump_syms_packages(self, state_dir, checkout_root):
+        '''
+        Install dump_syms.
+        '''
+        pass
+
     def install_toolchain_static_analysis(self, state_dir, checkout_root, toolchain_job):
         clang_tools_path = os.path.join(state_dir, 'clang-tools')
         if not os.path.exists(clang_tools_path):
             os.mkdir(clang_tools_path)
         self.install_toolchain_artifact(clang_tools_path, checkout_root, toolchain_job)
 
     def install_toolchain_artifact(self, state_dir, checkout_root, toolchain_job,
                                    no_unpack=False):
--- a/python/mozboot/mozboot/bootstrap.py
+++ b/python/mozboot/mozboot/bootstrap.py
@@ -395,16 +395,17 @@ class Bootstrapper(object):
         self.instance.ensure_node_packages(state_dir, checkout_root)
         if not self.instance.artifact_mode:
             self.instance.ensure_stylo_packages(state_dir, checkout_root)
             self.instance.ensure_clang_static_analysis_package(state_dir, checkout_root)
             self.instance.ensure_nasm_packages(state_dir, checkout_root)
             self.instance.ensure_sccache_packages(state_dir, checkout_root)
             self.instance.ensure_lucetc_packages(state_dir, checkout_root)
             self.instance.ensure_wasi_sysroot_packages(state_dir, checkout_root)
+            self.instance.ensure_dump_syms_packages(state_dir, checkout_root)
 
     def check_telemetry_opt_in(self, state_dir):
         # We can't prompt the user.
         if self.instance.no_interactive:
             return
         # Don't prompt if the user already has a setting for this value.
         if self.mach_context is not None and 'telemetry' in self.mach_context.settings.build:
             return
--- a/python/mozboot/mozboot/mozillabuild.py
+++ b/python/mozboot/mozboot/mozillabuild.py
@@ -131,16 +131,21 @@ class MozillaBuildBootstrapper(BaseBoots
     def ensure_node_packages(self, state_dir, checkout_root):
         from mozboot import node
         # We don't have native aarch64 node available, but aarch64 windows
         # runs x86 binaries, so just use the x86 packages for such hosts.
         node_artifact = node.WIN32 if is_aarch64_host() else node.WIN64
         self.install_toolchain_artifact(
             state_dir, checkout_root, node_artifact)
 
+    def ensure_dump_syms_packages(self, state_dir, checkout_root):
+        from mozboot import dump_syms
+
+        self.install_toolchain_artifact(state_dir, checkout_root, dump_syms.WIN64_DUMP_SYMS)
+
     def _update_package_manager(self):
         pass
 
     def run(self, command):
         subprocess.check_call(command, stdin=sys.stdin)
 
     def pip_install(self, *packages):
         pip_dir = os.path.join(os.environ['MOZILLABUILD'], 'python', 'Scripts', 'pip.exe')
--- a/python/mozbuild/mozbuild/action/dumpsymbols.py
+++ b/python/mozbuild/mozbuild/action/dumpsymbols.py
@@ -27,39 +27,34 @@ def dump_symbols(target, tracking_file, 
 
     # Build default args for symbolstore.py based on platform.
     sym_store_args = []
 
     # Find the `dump_syms` binary to use.
     dump_syms_bin = None
     dump_syms_binaries = []
 
-    # Prefer the `dump_syms` toolchain.
-    fetches_dir = os.environ.get('MOZ_FETCHES_DIR')
-    if fetches_dir:
-        dump_syms_binaries.append(
-            os.path.join(fetches_dir, 'dump_syms', 'dump_syms'))
+    default_bin = buildconfig.substs.get('DUMP_SYMS')
+    if default_bin:
+        dump_syms_binaries.append(default_bin)
 
     # Fallback to the in-tree breakpad version.
-    dump_syms_binaries.append(os.path.join(buildconfig.topobjdir,
-                                           'dist', 'host',
-                                           'bin', 'dump_syms'))
+    dump_syms_binaries.append(
+        os.path.join(buildconfig.topobjdir, 'dist', 'host', 'bin',
+                     'dump_syms' + buildconfig.substs['BIN_SUFFIX']))
 
-    for b in dump_syms_binaries:
-        dump_syms_bin = '%s%s' % (b, buildconfig.substs['BIN_SUFFIX'])
+    for dump_syms_bin in dump_syms_binaries:
         if os.path.exists(dump_syms_bin):
             break
 
     os_arch = buildconfig.substs['OS_ARCH']
     if os_arch == 'WINNT':
         sym_store_args.extend(['-c', '--vcs-info'])
         if os.environ.get('PDBSTR_PATH'):
             sym_store_args.append('-i')
-        os.environ['PATH'] = os.pathsep.join((buildconfig.substs['WIN_DIA_SDK_BIN_DIR'],
-                                              os.environ['PATH']))
     elif os_arch == 'Darwin':
         cpu = {
             'x86': 'i386',
         }.get(buildconfig.substs['TARGET_CPU'], buildconfig.substs['TARGET_CPU'])
         sym_store_args.extend(['-c', '-a', cpu, '--vcs-info'])
     elif os_arch == 'Linux':
         sym_store_args.extend(['-c', '--vcs-info'])
 
deleted file mode 100644
--- a/toolkit/crashreporter/google-breakpad/src/tools/windows/dump_syms/moz.build
+++ /dev/null
@@ -1,36 +0,0 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-HostProgram('dump_syms')
-
-HOST_SOURCES += [
-    '../../../common/windows/dia_util.cc',
-    '../../../common/windows/guid_string.cc',
-    '../../../common/windows/omap.cc',
-    '../../../common/windows/pdb_source_line_writer.cc',
-    '../../../common/windows/string_utils.cc',
-    'dump_syms.cc',
-]
-
-HOST_CXXFLAGS += [
-    '-O2',
-]
-
-if CONFIG['HOST_CC_TYPE'] == 'clang-cl':
-    HOST_CXXFLAGS += [
-        '-EHsc',
-        '-MD'
-    ]
-
-HOST_OS_LIBS += [
-    'dbghelp',
-    'diaguids',
-    'imagehlp',
-]
-
-LOCAL_INCLUDES += [
-    '../../..'
-]
--- a/toolkit/crashreporter/moz.build
+++ b/toolkit/crashreporter/moz.build
@@ -26,19 +26,16 @@ FINAL_LIBRARY = 'xul'
 if CONFIG['MOZ_CRASHREPORTER']:
     if CONFIG['OS_ARCH'] == 'WINNT':
         DIRS += [
             'google-breakpad/src/common',
             'google-breakpad/src/processor',
             'breakpad-windows-libxul',
         ]
 
-        if CONFIG['WIN_DIA_SDK_BIN_DIR']:
-            DIRS += ['google-breakpad/src/tools/windows/dump_syms']
-
         if CONFIG['MOZ_CRASHREPORTER_INJECTOR']:
             DIRS += ['breakpad-windows-standalone']
 
     elif CONFIG['OS_ARCH'] == 'Darwin':
         DIRS += [
             'breakpad-client',
             'breakpad-client/mac/crash_generation',
             'breakpad-client/mac/handler',
--- a/toolkit/crashreporter/tools/unit-symbolstore.py
+++ b/toolkit/crashreporter/tools/unit-symbolstore.py
@@ -472,22 +472,19 @@ class TestFunctional(HelperMixin, unitte
         # Bug 1608146.
         if buildconfig.substs.get('MOZ_CODE_COVERAGE'):
             self.skip_test = True
         self.topsrcdir = buildconfig.topsrcdir
         self.script_path = os.path.join(self.topsrcdir, 'toolkit',
                                         'crashreporter', 'tools',
                                         'symbolstore.py')
         if target_platform() == 'WINNT':
-            if buildconfig.substs['WIN_DIA_SDK_BIN_DIR']:
-                self.dump_syms = os.path.join(buildconfig.topobjdir,
-                                              'dist', 'host', 'bin',
-                                              'dump_syms.exe')
-            else:
-                self.skip_test = True
+            fetches_dir = os.environ.get('MOZ_FETCHES_DIR')
+            self.dump_syms = os.path.join(fetches_dir, 'dump_syms',
+                                          'dump_syms.exe')
             self.target_bin = os.path.join(buildconfig.topobjdir,
                                            'dist', 'bin',
                                            'firefox.exe')
         else:
             self.dump_syms = os.path.join(buildconfig.topobjdir,
                                           'dist', 'host', 'bin',
                                           'dump_syms')
             self.target_bin = os.path.join(buildconfig.topobjdir,
--- a/toolkit/moz.configure
+++ b/toolkit/moz.configure
@@ -1922,8 +1922,33 @@ set_define('MOZ_NEW_NOTIFICATION_STORE',
 
 @depends(milestone)
 def fogotype(milestone):
     if milestone.is_nightly:
         return True
 
 set_config('MOZ_FOGOTYPE', True, when=fogotype)
 set_define('MOZ_FOGOTYPE', True, when=fogotype)
+
+# dump_syms
+# ==============================================================
+
+option(env='DUMP_SYMS', nargs=1, when=compile_environment & target_is_windows,
+       help='Path to dump_syms')
+
+@depends('DUMP_SYMS', toolchain_search_path, when=compile_environment & target_is_windows)
+@checking('for dump_syms')
+@imports(_from='textwrap', _import='dedent')
+def dump_syms(dump_syms_override, toolchain_search_path):
+    if dump_syms_override:
+        return dump_syms_override[0]
+
+    program = find_program('dump_syms', paths=toolchain_search_path)
+
+    if program:
+        return program
+
+    raise FatalCheckError(dedent('''\
+    Cannot find dump_syms. Please run `mach bootstrap`, `cargo install dump_syms`,
+    ensure that `dump_syms` is on your PATH, or point at an executable with `DUMP_SYMS`.
+    '''))
+
+set_config('DUMP_SYMS', dump_syms)
--- a/tools/crashreporter/app.mozbuild
+++ b/tools/crashreporter/app.mozbuild
@@ -17,16 +17,8 @@ if CONFIG['OS_ARCH'] == 'Linux':
         '/toolkit/crashreporter/google-breakpad/src/tools/linux/dump_syms',
     ]
 
 if CONFIG['OS_ARCH'] == 'Darwin':
     DIRS += [
         '/toolkit/crashreporter/google-breakpad/src/common/mac',
         '/toolkit/crashreporter/google-breakpad/src/tools/mac/dump_syms',
     ]
-
-# While the Linux and Mac dump_syms can be cross-built, the Windows one can't,
-# and at the moment, it's a host tool, so only build it when the host is
-# Windows.
-if CONFIG['HOST_OS_ARCH'] == 'WINNT':
-    DIRS += [
-        '/toolkit/crashreporter/google-breakpad/src/tools/windows/dump_syms',
-    ]