Bug 1337986 - Dump symbols during the compile tier. r=ted
authorChris Manchester <cmanchester@mozilla.com>
Fri, 10 Feb 2017 08:34:08 -0800
changeset 560224 994b97c9de3741e5bea37a8baf1573ded2323336
parent 560223 17fc25af0768621dec049d39bd87bc682fc16d8e
child 560225 4b2125df04a83ee3beea9417b6077b3d55ac93db
push id53365
push userjichen@mozilla.com
push dateTue, 11 Apr 2017 08:35:12 +0000
reviewersted
bugs1337986
milestone55.0a1
Bug 1337986 - Dump symbols during the compile tier. r=ted This commit moves symbol dumping to the compile tier, to be run via "syms" targets. Tracking files are used for the sake of incremental builds, because dump_syms may genearate multiple outputs whose paths are not known ahead of time. Minimal changes to symbolstore.py are made here. More extensive simplifications will be made in a future commit on the basis of symbolstore.py handling one file at a time. MozReview-Commit-ID: 3mOP8A6Y7iM
Makefile.in
config/rules.mk
python/mozbuild/mozbuild/action/dumpsymbols.py
toolkit/crashreporter/tools/symbolstore.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -243,74 +243,39 @@ default::
 	@echo "===SCCACHE STATS==="
 	-$(CCACHE) --show-stats
 	@echo "==================="
 endif
 
 distclean::
 	$(RM) $(DIST_GARBAGE)
 
-ifeq ($(OS_ARCH),WINNT)
-# we want to copy PDB files on Windows
-MAKE_SYM_STORE_ARGS := -c --vcs-info
-ifdef PDBSTR_PATH
-MAKE_SYM_STORE_ARGS += -i
-endif
-ifdef MSVC_HAS_DIA_SDK
-DUMP_SYMS_BIN ?= $(DIST)/host/bin/dump_syms.exe
-else
-DUMP_SYMS_BIN ?= $(topsrcdir)/toolkit/crashreporter/tools/win32/dump_syms_vc$(_MSC_VER).exe
-endif
-# PDB files don't get moved to dist, so we need to scan the whole objdir
-MAKE_SYM_STORE_PATH := .
-endif
-ifeq ($(OS_ARCH),Darwin)
-# need to pass arch flags for universal builds
-MAKE_SYM_STORE_ARGS := -c -a $(OS_TEST) --vcs-info
-MAKE_SYM_STORE_PATH := $(DIST)/bin
-DUMP_SYMS_BIN ?= $(DIST)/host/bin/dump_syms
-endif
-ifeq (,$(filter-out Linux SunOS,$(OS_ARCH)))
-MAKE_SYM_STORE_ARGS := -c --vcs-info
-DUMP_SYMS_BIN ?= $(DIST)/host/bin/dump_syms
-MAKE_SYM_STORE_PATH := $(DIST)/bin
-endif
-MAKE_SYM_STORE_ARGS += --install-manifest=$(DEPTH)/_build_manifests/install/dist_include,$(DIST)/include
-
-SYM_STORE_SOURCE_DIRS := $(topsrcdir)
-
 ifdef MOZ_CRASHREPORTER
 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
 
 endif
 
-.PHONY: generatesymbols
-generatesymbols:
-	echo building symbol store
-	$(RM) -r $(DIST)/crashreporter-symbols
-	$(RM) '$(DIST)/$(SYMBOL_ARCHIVE_BASENAME).zip'
-	$(RM) '$(DIST)/$(SYMBOL_FULL_ARCHIVE_BASENAME).zip'
-	$(NSINSTALL) -D $(DIST)/crashreporter-symbols
-	OBJCOPY='$(OBJCOPY)' \
-	$(PYTHON) $(topsrcdir)/toolkit/crashreporter/tools/symbolstore.py \
-	  $(MAKE_SYM_STORE_ARGS)                                          \
-	  $(foreach dir,$(SYM_STORE_SOURCE_DIRS),-s $(dir))               \
-	  $(DUMP_SYMS_BIN)                                                \
-	  $(DIST)/crashreporter-symbols                                   \
-	  $(MAKE_SYM_STORE_PATH)
+.PHONY: prepsymbolsarchive
+prepsymbolsarchive:
 	echo packing symbols
 	$(NSINSTALL) -D $(DIST)/$(PKG_PATH)
 
+ifndef MOZ_AUTOMATION
+prepsymbolsarchive: recurse_syms
+endif
+
 .PHONY: symbolsfullarchive
-symbolsfullarchive: generatesymbols
+symbolsfullarchive: prepsymbolsarchive
+	$(RM) '$(DIST)/$(SYMBOL_FULL_ARCHIVE_BASENAME).zip'
 	cd $(DIST)/crashreporter-symbols && \
           zip -r5D '../$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip' . -x '*test*' -x '*Test*'
 
 .PHONY: symbolsarchive
-symbolsarchive: generatesymbols
+symbolsarchive: prepsymbolsarchive
+	$(RM) '$(DIST)/$(SYMBOL_ARCHIVE_BASENAME).zip'
 	cd $(DIST)/crashreporter-symbols && \
           zip -r5D '../$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip' . -i '*.sym'
 
 ifdef MOZ_CRASHREPORTER
 buildsymbols: symbolsfullarchive symbolsarchive
 else
 buildsymbols:
 endif
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -875,16 +875,36 @@ crate_src_libdep = $(call mk_global_crat
 ifdef ASFILES
 # The AS_DASH_C_FLAG is needed cause not all assemblers (Solaris) accept
 # a '-c' flag.
 $(ASOBJS):
 	$(REPORT_BUILD_VERBOSE)
 	$(AS) $(ASOUTOPTION)$@ $(ASFLAGS) $($(notdir $<)_FLAGS) $(AS_DASH_C_FLAG) $(_VPATH_SRCS)
 endif
 
+define syms_template
+syms:: $(2)
+$(2): $(1)
+	$$(call py_action,dumpsymbols,$$(abspath $$<) $$(abspath $$@))
+endef
+
+ifndef MOZ_PROFILE_GENERATE
+ifneq (,$(filter $(DIST)/bin%,$(FINAL_TARGET)))
+DUMP_SYMS_TARGETS := $(SHARED_LIBRARY) $(PROGRAM) $(SIMPLE_PROGRAMS)
+endif
+endif
+
+ifdef MOZ_AUTOMATION
+ifeq (,$(filter 1,$(MOZ_AUTOMATION_BUILD_SYMBOLS)))
+DUMP_SYMS_TARGETS :=
+endif
+endif
+
+$(foreach file,$(DUMP_SYMS_TARGETS),$(eval $(call syms_template,$(file),$(file)_syms.track)))
+
 ifdef MOZ_RUST
 cargo_host_flag := --target=$(RUST_HOST_TARGET)
 cargo_target_flag := --target=$(RUST_TARGET)
 
 # Permit users to pass flags to cargo from their mozconfigs (e.g. --color=always).
 cargo_build_flags = $(CARGOFLAGS)
 ifndef MOZ_DEBUG
 cargo_build_flags += --release
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/action/dumpsymbols.py
@@ -0,0 +1,79 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import argparse
+import buildconfig
+import subprocess
+import shutil
+import sys
+import os
+
+def dump_symbols(target, tracking_file):
+    # Our tracking file, if present, will contain path(s) to the previously generated
+    # symbols. Remove them in this case so we don't simply accumulate old symbols
+    # during incremental builds.
+    if os.path.isfile(os.path.normpath(tracking_file)):
+        with open(tracking_file, 'r') as fh:
+            files = fh.read().splitlines()
+        dirs = set(os.path.dirname(f) for f in files)
+        for d in dirs:
+            shutil.rmtree(os.path.join(buildconfig.topobjdir, 'dist',
+                                       'crashreporter-symbols', d),
+                          ignore_errors=True)
+
+    # Build default args for symbolstore.py based on platform.
+    sym_store_args = []
+
+    dump_syms_bin = os.path.join(buildconfig.topobjdir,
+                                 'dist', 'host',
+                                 'bin', 'dump_syms')
+    dump_syms_bin = '%s%s' % (dump_syms_bin, buildconfig.substs['BIN_SUFFIX'])
+
+    os_arch = buildconfig.substs['OS_ARCH']
+    if os_arch == 'WINNT':
+        sym_store_args.extend(['-c', '--vcs-info'])
+        if os.environ.get('PDBSTR_PATH'):
+            sym_store_args.append('-i')
+    elif os_arch == 'Darwin':
+        sym_store_args.extend(['-c', '-a', buildconfig.substs['OS_TEST'], '--vcs-info'])
+    elif os_arch == 'Linux':
+        sym_store_args.extend(['-c', '--vcs-info'])
+
+    sym_store_args.append('--install-manifest=%s,%s' % (os.path.join(buildconfig.topobjdir,
+                                                                     '_build_manifests',
+                                                                     'install',
+                                                                     'dist_include'),
+                                                        os.path.join(buildconfig.topobjdir,
+                                                                     'dist',
+                                                                     'include')))
+    objcopy = buildconfig.substs.get('OBJCOPY')
+    if objcopy:
+        os.environ['OBJCOPY'] = objcopy
+
+    args = ([buildconfig.substs['PYTHON'], os.path.join(buildconfig.topsrcdir, 'toolkit',
+                                                       'crashreporter', 'tools', 'symbolstore.py')] +
+            sym_store_args +
+            ['-s', buildconfig.topsrcdir, dump_syms_bin, os.path.join(buildconfig.topobjdir,
+                                                                      'dist',
+                                                                      'crashreporter-symbols'),
+             os.path.abspath(target)])
+    print('Running: %s' % ' '.join(args))
+    out_files = subprocess.check_output(args)
+    with open(tracking_file, 'w') as fh:
+        fh.write(out_files)
+        fh.flush()
+
+def main(argv):
+    if len(argv) != 2:
+        print("Usage: dumpsymbols.py <library or program> <tracking file>",
+              file=sys.stderr)
+        return 1
+
+    return dump_symbols(*argv)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))
--- a/toolkit/crashreporter/tools/symbolstore.py
+++ b/toolkit/crashreporter/tools/symbolstore.py
@@ -572,17 +572,17 @@ class Dumper:
         for f in sorted(files, key=os.path.getsize, reverse=True):
             self.ProcessFiles((f,))
 
     def get_files_to_process(self, file_or_dir):
         """Generate the files to process from an input."""
         if os.path.isdir(file_or_dir) and not self.ShouldSkipDir(file_or_dir):
             for f in self.get_files_to_process_in_dir(file_or_dir):
                 yield f
-        elif os.path.isfile(file_or_dir):
+        elif os.path.isfile(file_or_dir) and self.ShouldProcess(os.path.abspath(file_or_dir)):
             yield file_or_dir
 
     def get_files_to_process_in_dir(self, path):
         """Generate the files to process in a directory.
 
         Valid files are are determined by calling ShouldProcess.
         """
         for root, dirs, files in os.walk(path):