Bug 1337986 - Dump symbols during the compile tier. draft
authorChris Manchester <cmanchester@mozilla.com>
Fri, 10 Feb 2017 08:34:08 -0800
changeset 555118 227b5af9179ff3849c951c1a624ede13e9e7b3ce
parent 555117 92717b884006933cdf79446e7b06381d70439a64
child 555119 3de3c40836b17356f80230ee81fd9ff09389ced4
push id52161
push userbmo:cmanchester@mozilla.com
push dateMon, 03 Apr 2017 17:35:35 +0000
bugs1337986
milestone55.0a1
Bug 1337986 - Dump symbols during the compile tier. This commit moves symbol dumping to the compile tier, to be run via "syms" targets. Tracking files are used for the sake of incremental builds, because dump_syms may genearate multiple outputs whose paths are not known ahead of time. Minimal changes to symbolstore.py are made here. More extensive simplifications will be made in a future commit on the basis of symbolstore.py handling one file at a time. MozReview-Commit-ID: 3mOP8A6Y7iM
Makefile.in
config/rules.mk
python/mozbuild/mozbuild/action/dumpsymbols.py
toolkit/crashreporter/tools/symbolstore.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -243,74 +243,39 @@ default::
 	@echo "===SCCACHE STATS==="
 	-$(CCACHE) --show-stats
 	@echo "==================="
 endif
 
 distclean::
 	$(RM) $(DIST_GARBAGE)
 
-ifeq ($(OS_ARCH),WINNT)
-# we want to copy PDB files on Windows
-MAKE_SYM_STORE_ARGS := -c --vcs-info
-ifdef PDBSTR_PATH
-MAKE_SYM_STORE_ARGS += -i
-endif
-ifdef MSVC_HAS_DIA_SDK
-DUMP_SYMS_BIN ?= $(DIST)/host/bin/dump_syms.exe
-else
-DUMP_SYMS_BIN ?= $(topsrcdir)/toolkit/crashreporter/tools/win32/dump_syms_vc$(_MSC_VER).exe
-endif
-# PDB files don't get moved to dist, so we need to scan the whole objdir
-MAKE_SYM_STORE_PATH := .
-endif
-ifeq ($(OS_ARCH),Darwin)
-# need to pass arch flags for universal builds
-MAKE_SYM_STORE_ARGS := -c -a $(OS_TEST) --vcs-info
-MAKE_SYM_STORE_PATH := $(DIST)/bin
-DUMP_SYMS_BIN ?= $(DIST)/host/bin/dump_syms
-endif
-ifeq (,$(filter-out Linux SunOS,$(OS_ARCH)))
-MAKE_SYM_STORE_ARGS := -c --vcs-info
-DUMP_SYMS_BIN ?= $(DIST)/host/bin/dump_syms
-MAKE_SYM_STORE_PATH := $(DIST)/bin
-endif
-MAKE_SYM_STORE_ARGS += --install-manifest=$(DEPTH)/_build_manifests/install/dist_include,$(DIST)/include
-
-SYM_STORE_SOURCE_DIRS := $(topsrcdir)
-
 ifdef MOZ_CRASHREPORTER
 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
 
 endif
 
-.PHONY: generatesymbols
-generatesymbols:
-	echo building symbol store
-	$(RM) -r $(DIST)/crashreporter-symbols
-	$(RM) '$(DIST)/$(SYMBOL_ARCHIVE_BASENAME).zip'
-	$(RM) '$(DIST)/$(SYMBOL_FULL_ARCHIVE_BASENAME).zip'
-	$(NSINSTALL) -D $(DIST)/crashreporter-symbols
-	OBJCOPY='$(OBJCOPY)' \
-	$(PYTHON) $(topsrcdir)/toolkit/crashreporter/tools/symbolstore.py \
-	  $(MAKE_SYM_STORE_ARGS)                                          \
-	  $(foreach dir,$(SYM_STORE_SOURCE_DIRS),-s $(dir))               \
-	  $(DUMP_SYMS_BIN)                                                \
-	  $(DIST)/crashreporter-symbols                                   \
-	  $(MAKE_SYM_STORE_PATH)
+.PHONY: prepsymbolsarchive
+prepsymbolsarchive:
 	echo packing symbols
 	$(NSINSTALL) -D $(DIST)/$(PKG_PATH)
 
+ifndef MOZ_AUTOMATION
+prepsymbolsarchive: recurse_syms
+endif
+
 .PHONY: symbolsfullarchive
-symbolsfullarchive: generatesymbols
+symbolsfullarchive: prepsymbolsarchive
+	$(RM) '$(DIST)/$(SYMBOL_FULL_ARCHIVE_BASENAME).zip'
 	cd $(DIST)/crashreporter-symbols && \
           zip -r5D '../$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip' . -x '*test*' -x '*Test*'
 
 .PHONY: symbolsarchive
-symbolsarchive: generatesymbols
+symbolsarchive: prepsymbolsarchive
+	$(RM) '$(DIST)/$(SYMBOL_ARCHIVE_BASENAME).zip'
 	cd $(DIST)/crashreporter-symbols && \
           zip -r5D '../$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip' . -i '*.sym'
 
 ifdef MOZ_CRASHREPORTER
 buildsymbols: symbolsfullarchive symbolsarchive
 else
 buildsymbols:
 endif
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -900,16 +900,32 @@ crate_src_libdep = $(call mk_global_crat
 ifdef ASFILES
 # The AS_DASH_C_FLAG is needed cause not all assemblers (Solaris) accept
 # a '-c' flag.
 $(ASOBJS):
 	$(REPORT_BUILD_VERBOSE)
 	$(AS) $(ASOUTOPTION)$@ $(ASFLAGS) $($(notdir $<)_FLAGS) $(AS_DASH_C_FLAG) $(_VPATH_SRCS)
 endif
 
+define syms_template
+syms:: $(2)
+$(2): $(1)
+	$$(call py_action,dumpsymbols,$$(abspath $$<) $$(abspath $$@))
+endef
+
+ifndef MOZ_PROFILE_GENERATE
+# ifneq (_,$(filter 1,$(MOZ_AUTOMATION_BUILD_SYMBOLS))_$(filter syms,$(MAKECMDGOALS)))
+ifneq (,$(filter $(DIST)/bin%,$(FINAL_TARGET)))
+DUMP_SYMS_TARGETS := $(SHARED_LIBRARY) $(PROGRAM) $(SIMPLE_PROGRAMS)
+endif
+# endif
+endif
+
+$(foreach file,$(DUMP_SYMS_TARGETS),$(eval $(call syms_template,$(file),$(file)_syms.track)))
+
 ifdef MOZ_RUST
 cargo_host_flag := --target=$(RUST_HOST_TARGET)
 cargo_target_flag := --target=$(RUST_TARGET)
 
 # Permit users to pass flags to cargo from their mozconfigs (e.g. --color=always).
 cargo_build_flags = $(CARGOFLAGS)
 ifndef MOZ_DEBUG
 cargo_build_flags += --release
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/action/dumpsymbols.py
@@ -0,0 +1,84 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import argparse
+import buildconfig
+import subprocess
+import shutil
+import sys
+import os
+
+def dump_symbols(target, tracking_file):
+    # Our tracking file, if present, will contain path(s) to the previously generated
+    # symbols. Remove them in this case so we don't simply accumulate old symbols
+    # during incremental builds.
+    if os.path.isfile(os.path.normpath(tracking_file)):
+        with open(tracking_file, 'r') as fh:
+            files = fh.read().splitlines()
+        dirs = set(os.path.dirname(f) for f in files)
+        for d in dirs:
+            shutil.rmtree(os.path.join(buildconfig.topobjdir, 'dist',
+                                       'crashreporter-symbols', d),
+                          ignore_errors=True)
+
+    # Build default args for symbolstore.py based on platform.
+    sym_store_args = []
+
+    dump_syms_bin = os.path.join(buildconfig.topobjdir,
+                                 'dist', 'host',
+                                 'bin', 'dump_syms')
+
+    os_arch = buildconfig.substs['OS_ARCH']
+    if os_arch == 'WINNT':
+        sym_store_args.extend(['-c', '--vcs-info'])
+        if os.environ.get('PDBSTR_PATH'):
+            sym_store_args.append('-i')
+        if buildconfig.substs.get('MSVC_HAS_DIA_SDK'):
+            dump_syms_bin = "%s.exe" % dump_syms_bin
+        else:
+            dump_syms_bin = os.path.join(buildconfig.topsrcdir, 'toolkit',
+                                         'crashreporter', 'tools', 'win32',
+                                         'dump_syms_vc%s.exe' % buildconfig.substs['_MSC_VER'])
+    elif os_arch == 'Darwin':
+        sym_store_args.extend(['-c', '-a', buildconfig.substs['OS_TEST'], '--vcs-info'])
+    elif os_arch in ('Linux', 'SunOS'):
+        sym_store_args.extend(['-c', '--vcs-info'])
+
+    sym_store_args.append('--install-manifest=%s,%s' % (os.path.join(buildconfig.topobjdir,
+                                                                     '_build_manifests',
+                                                                     'install',
+                                                                     'dist_include'),
+                                                        os.path.join(buildconfig.topobjdir,
+                                                                     'dist',
+                                                                     'include')))
+    objcopy = buildconfig.substs.get('OBJCOPY')
+    if objcopy:
+        os.environ['OBJCOPY'] = objcopy
+
+    args = ([buildconfig.substs['PYTHON'], os.path.join(buildconfig.topsrcdir, 'toolkit',
+                                                       'crashreporter', 'tools', 'symbolstore.py')] +
+            sym_store_args +
+            ['-s', buildconfig.topsrcdir, dump_syms_bin, os.path.join(buildconfig.topobjdir,
+                                                                      'dist',
+                                                                      'crashreporter-symbols'),
+             os.path.abspath(target)])
+    print('Running: %s' % ' '.join(args))
+    out_files = subprocess.check_output(args)
+    with open(tracking_file, 'w') as fh:
+        fh.write(out_files)
+        fh.flush()
+
+def main(argv):
+    if len(argv) != 2:
+        print("Usage: dumpsymbols.py <library or program> <tracking file>",
+              file=sys.stderr)
+        return 1
+
+    return dump_symbols(*argv)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))
--- a/toolkit/crashreporter/tools/symbolstore.py
+++ b/toolkit/crashreporter/tools/symbolstore.py
@@ -572,17 +572,17 @@ class Dumper:
         for f in sorted(files, key=os.path.getsize, reverse=True):
             self.ProcessFiles((f,))
 
     def get_files_to_process(self, file_or_dir):
         """Generate the files to process from an input."""
         if os.path.isdir(file_or_dir) and not self.ShouldSkipDir(file_or_dir):
             for f in self.get_files_to_process_in_dir(file_or_dir):
                 yield f
-        elif os.path.isfile(file_or_dir):
+        elif os.path.isfile(file_or_dir) and self.ShouldProcess(os.path.abspath(file_or_dir)):
             yield file_or_dir
 
     def get_files_to_process_in_dir(self, path):
         """Generate the files to process in a directory.
 
         Valid files are are determined by calling ShouldProcess.
         """
         for root, dirs, files in os.walk(path):