Bug 1337986 - Dump symbols during the compile tier. r=ted
This commit moves symbol dumping to the compile tier, to be run via "syms"
targets. Tracking files are used for the sake of incremental builds, because
dump_syms may genearate multiple outputs whose paths are not known ahead of
time.
Minimal changes to symbolstore.py are made here. More extensive
simplifications will be made in a future commit on the basis of symbolstore.py
handling one file at a time.
MozReview-Commit-ID: 3mOP8A6Y7iM
--- a/Makefile.in
+++ b/Makefile.in
@@ -243,74 +243,39 @@ default::
@echo "===SCCACHE STATS==="
-$(CCACHE) --show-stats
@echo "==================="
endif
distclean::
$(RM) $(DIST_GARBAGE)
-ifeq ($(OS_ARCH),WINNT)
-# we want to copy PDB files on Windows
-MAKE_SYM_STORE_ARGS := -c --vcs-info
-ifdef PDBSTR_PATH
-MAKE_SYM_STORE_ARGS += -i
-endif
-ifdef MSVC_HAS_DIA_SDK
-DUMP_SYMS_BIN ?= $(DIST)/host/bin/dump_syms.exe
-else
-DUMP_SYMS_BIN ?= $(topsrcdir)/toolkit/crashreporter/tools/win32/dump_syms_vc$(_MSC_VER).exe
-endif
-# PDB files don't get moved to dist, so we need to scan the whole objdir
-MAKE_SYM_STORE_PATH := .
-endif
-ifeq ($(OS_ARCH),Darwin)
-# need to pass arch flags for universal builds
-MAKE_SYM_STORE_ARGS := -c -a $(OS_TEST) --vcs-info
-MAKE_SYM_STORE_PATH := $(DIST)/bin
-DUMP_SYMS_BIN ?= $(DIST)/host/bin/dump_syms
-endif
-ifeq (,$(filter-out Linux SunOS,$(OS_ARCH)))
-MAKE_SYM_STORE_ARGS := -c --vcs-info
-DUMP_SYMS_BIN ?= $(DIST)/host/bin/dump_syms
-MAKE_SYM_STORE_PATH := $(DIST)/bin
-endif
-MAKE_SYM_STORE_ARGS += --install-manifest=$(DEPTH)/_build_manifests/install/dist_include,$(DIST)/include
-
-SYM_STORE_SOURCE_DIRS := $(topsrcdir)
-
ifdef MOZ_CRASHREPORTER
include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
endif
-.PHONY: generatesymbols
-generatesymbols:
- echo building symbol store
- $(RM) -r $(DIST)/crashreporter-symbols
- $(RM) '$(DIST)/$(SYMBOL_ARCHIVE_BASENAME).zip'
- $(RM) '$(DIST)/$(SYMBOL_FULL_ARCHIVE_BASENAME).zip'
- $(NSINSTALL) -D $(DIST)/crashreporter-symbols
- OBJCOPY='$(OBJCOPY)' \
- $(PYTHON) $(topsrcdir)/toolkit/crashreporter/tools/symbolstore.py \
- $(MAKE_SYM_STORE_ARGS) \
- $(foreach dir,$(SYM_STORE_SOURCE_DIRS),-s $(dir)) \
- $(DUMP_SYMS_BIN) \
- $(DIST)/crashreporter-symbols \
- $(MAKE_SYM_STORE_PATH)
+.PHONY: prepsymbolsarchive
+prepsymbolsarchive:
echo packing symbols
$(NSINSTALL) -D $(DIST)/$(PKG_PATH)
+ifndef MOZ_AUTOMATION
+prepsymbolsarchive: recurse_syms
+endif
+
.PHONY: symbolsfullarchive
-symbolsfullarchive: generatesymbols
+symbolsfullarchive: prepsymbolsarchive
+ $(RM) '$(DIST)/$(SYMBOL_FULL_ARCHIVE_BASENAME).zip'
cd $(DIST)/crashreporter-symbols && \
zip -r5D '../$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip' . -x '*test*' -x '*Test*'
.PHONY: symbolsarchive
-symbolsarchive: generatesymbols
+symbolsarchive: prepsymbolsarchive
+ $(RM) '$(DIST)/$(SYMBOL_ARCHIVE_BASENAME).zip'
cd $(DIST)/crashreporter-symbols && \
zip -r5D '../$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip' . -i '*.sym'
ifdef MOZ_CRASHREPORTER
buildsymbols: symbolsfullarchive symbolsarchive
else
buildsymbols:
endif
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -875,16 +875,36 @@ crate_src_libdep = $(call mk_global_crat
ifdef ASFILES
# The AS_DASH_C_FLAG is needed cause not all assemblers (Solaris) accept
# a '-c' flag.
$(ASOBJS):
$(REPORT_BUILD_VERBOSE)
$(AS) $(ASOUTOPTION)$@ $(ASFLAGS) $($(notdir $<)_FLAGS) $(AS_DASH_C_FLAG) $(_VPATH_SRCS)
endif
+define syms_template
+syms:: $(2)
+$(2): $(1)
+ $$(call py_action,dumpsymbols,$$(abspath $$<) $$(abspath $$@))
+endef
+
+ifndef MOZ_PROFILE_GENERATE
+ifneq (,$(filter $(DIST)/bin%,$(FINAL_TARGET)))
+DUMP_SYMS_TARGETS := $(SHARED_LIBRARY) $(PROGRAM) $(SIMPLE_PROGRAMS)
+endif
+endif
+
+ifdef MOZ_AUTOMATION
+ifeq (,$(filter 1,$(MOZ_AUTOMATION_BUILD_SYMBOLS)))
+DUMP_SYMS_TARGETS :=
+endif
+endif
+
+$(foreach file,$(DUMP_SYMS_TARGETS),$(eval $(call syms_template,$(file),$(file)_syms.track)))
+
ifdef MOZ_RUST
cargo_host_flag := --target=$(RUST_HOST_TARGET)
cargo_target_flag := --target=$(RUST_TARGET)
# Permit users to pass flags to cargo from their mozconfigs (e.g. --color=always).
cargo_build_flags = $(CARGOFLAGS)
ifndef MOZ_DEBUG
cargo_build_flags += --release
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/action/dumpsymbols.py
@@ -0,0 +1,79 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import argparse
+import buildconfig
+import subprocess
+import shutil
+import sys
+import os
+
+def dump_symbols(target, tracking_file):
+ # Our tracking file, if present, will contain path(s) to the previously generated
+ # symbols. Remove them in this case so we don't simply accumulate old symbols
+ # during incremental builds.
+ if os.path.isfile(os.path.normpath(tracking_file)):
+ with open(tracking_file, 'r') as fh:
+ files = fh.read().splitlines()
+ dirs = set(os.path.dirname(f) for f in files)
+ for d in dirs:
+ shutil.rmtree(os.path.join(buildconfig.topobjdir, 'dist',
+ 'crashreporter-symbols', d),
+ ignore_errors=True)
+
+ # Build default args for symbolstore.py based on platform.
+ sym_store_args = []
+
+ dump_syms_bin = os.path.join(buildconfig.topobjdir,
+ 'dist', 'host',
+ 'bin', 'dump_syms')
+ dump_syms_bin = '%s%s' % (dump_syms_bin, buildconfig.substs['BIN_SUFFIX'])
+
+ os_arch = buildconfig.substs['OS_ARCH']
+ if os_arch == 'WINNT':
+ sym_store_args.extend(['-c', '--vcs-info'])
+ if os.environ.get('PDBSTR_PATH'):
+ sym_store_args.append('-i')
+ elif os_arch == 'Darwin':
+ sym_store_args.extend(['-c', '-a', buildconfig.substs['OS_TEST'], '--vcs-info'])
+ elif os_arch == 'Linux':
+ sym_store_args.extend(['-c', '--vcs-info'])
+
+ sym_store_args.append('--install-manifest=%s,%s' % (os.path.join(buildconfig.topobjdir,
+ '_build_manifests',
+ 'install',
+ 'dist_include'),
+ os.path.join(buildconfig.topobjdir,
+ 'dist',
+ 'include')))
+ objcopy = buildconfig.substs.get('OBJCOPY')
+ if objcopy:
+ os.environ['OBJCOPY'] = objcopy
+
+ args = ([buildconfig.substs['PYTHON'], os.path.join(buildconfig.topsrcdir, 'toolkit',
+ 'crashreporter', 'tools', 'symbolstore.py')] +
+ sym_store_args +
+ ['-s', buildconfig.topsrcdir, dump_syms_bin, os.path.join(buildconfig.topobjdir,
+ 'dist',
+ 'crashreporter-symbols'),
+ os.path.abspath(target)])
+ print('Running: %s' % ' '.join(args))
+ out_files = subprocess.check_output(args)
+ with open(tracking_file, 'w') as fh:
+ fh.write(out_files)
+ fh.flush()
+
+def main(argv):
+ if len(argv) != 2:
+ print("Usage: dumpsymbols.py <library or program> <tracking file>",
+ file=sys.stderr)
+ return 1
+
+ return dump_symbols(*argv)
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:]))
--- a/toolkit/crashreporter/tools/symbolstore.py
+++ b/toolkit/crashreporter/tools/symbolstore.py
@@ -572,17 +572,17 @@ class Dumper:
for f in sorted(files, key=os.path.getsize, reverse=True):
self.ProcessFiles((f,))
def get_files_to_process(self, file_or_dir):
"""Generate the files to process from an input."""
if os.path.isdir(file_or_dir) and not self.ShouldSkipDir(file_or_dir):
for f in self.get_files_to_process_in_dir(file_or_dir):
yield f
- elif os.path.isfile(file_or_dir):
+ elif os.path.isfile(file_or_dir) and self.ShouldProcess(os.path.abspath(file_or_dir)):
yield file_or_dir
def get_files_to_process_in_dir(self, path):
"""Generate the files to process in a directory.
Valid files are are determined by calling ShouldProcess.
"""
for root, dirs, files in os.walk(path):