Bug 1337986 - Dump symbols during the compile tier. r=ted
authorChris Manchester <cmanchester@mozilla.com>
Fri, 10 Feb 2017 08:34:08 -0800
changeset 352195 994b97c9de3741e5bea37a8baf1573ded2323336
parent 352194 17fc25af0768621dec049d39bd87bc682fc16d8e
child 352196 4b2125df04a83ee3beea9417b6077b3d55ac93db
push id40428
push usercmanchester@mozilla.com
push dateMon, 10 Apr 2017 17:35:20 +0000
treeherderautoland@7086da6987e0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1337986
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1337986 - Dump symbols during the compile tier. r=ted This commit moves symbol dumping to the compile tier, to be run via "syms" targets. Tracking files are used for the sake of incremental builds, because dump_syms may genearate multiple outputs whose paths are not known ahead of time. Minimal changes to symbolstore.py are made here. More extensive simplifications will be made in a future commit on the basis of symbolstore.py handling one file at a time. MozReview-Commit-ID: 3mOP8A6Y7iM
Makefile.in
config/rules.mk
python/mozbuild/mozbuild/action/dumpsymbols.py
toolkit/crashreporter/tools/symbolstore.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -243,74 +243,39 @@ default::
 	@echo "===SCCACHE STATS==="
 	-$(CCACHE) --show-stats
 	@echo "==================="
 endif
 
 distclean::
 	$(RM) $(DIST_GARBAGE)
 
-ifeq ($(OS_ARCH),WINNT)
-# we want to copy PDB files on Windows
-MAKE_SYM_STORE_ARGS := -c --vcs-info
-ifdef PDBSTR_PATH
-MAKE_SYM_STORE_ARGS += -i
-endif
-ifdef MSVC_HAS_DIA_SDK
-DUMP_SYMS_BIN ?= $(DIST)/host/bin/dump_syms.exe
-else
-DUMP_SYMS_BIN ?= $(topsrcdir)/toolkit/crashreporter/tools/win32/dump_syms_vc$(_MSC_VER).exe
-endif
-# PDB files don't get moved to dist, so we need to scan the whole objdir
-MAKE_SYM_STORE_PATH := .
-endif
-ifeq ($(OS_ARCH),Darwin)
-# need to pass arch flags for universal builds
-MAKE_SYM_STORE_ARGS := -c -a $(OS_TEST) --vcs-info
-MAKE_SYM_STORE_PATH := $(DIST)/bin
-DUMP_SYMS_BIN ?= $(DIST)/host/bin/dump_syms
-endif
-ifeq (,$(filter-out Linux SunOS,$(OS_ARCH)))
-MAKE_SYM_STORE_ARGS := -c --vcs-info
-DUMP_SYMS_BIN ?= $(DIST)/host/bin/dump_syms
-MAKE_SYM_STORE_PATH := $(DIST)/bin
-endif
-MAKE_SYM_STORE_ARGS += --install-manifest=$(DEPTH)/_build_manifests/install/dist_include,$(DIST)/include
-
-SYM_STORE_SOURCE_DIRS := $(topsrcdir)
-
 ifdef MOZ_CRASHREPORTER
 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
 
 endif
 
-.PHONY: generatesymbols
-generatesymbols:
-	echo building symbol store
-	$(RM) -r $(DIST)/crashreporter-symbols
-	$(RM) '$(DIST)/$(SYMBOL_ARCHIVE_BASENAME).zip'
-	$(RM) '$(DIST)/$(SYMBOL_FULL_ARCHIVE_BASENAME).zip'
-	$(NSINSTALL) -D $(DIST)/crashreporter-symbols
-	OBJCOPY='$(OBJCOPY)' \
-	$(PYTHON) $(topsrcdir)/toolkit/crashreporter/tools/symbolstore.py \
-	  $(MAKE_SYM_STORE_ARGS)                                          \
-	  $(foreach dir,$(SYM_STORE_SOURCE_DIRS),-s $(dir))               \
-	  $(DUMP_SYMS_BIN)                                                \
-	  $(DIST)/crashreporter-symbols                                   \
-	  $(MAKE_SYM_STORE_PATH)
+.PHONY: prepsymbolsarchive
+prepsymbolsarchive:
 	echo packing symbols
 	$(NSINSTALL) -D $(DIST)/$(PKG_PATH)
 
+ifndef MOZ_AUTOMATION
+prepsymbolsarchive: recurse_syms
+endif
+
 .PHONY: symbolsfullarchive
-symbolsfullarchive: generatesymbols
+symbolsfullarchive: prepsymbolsarchive
+	$(RM) '$(DIST)/$(SYMBOL_FULL_ARCHIVE_BASENAME).zip'
 	cd $(DIST)/crashreporter-symbols && \
           zip -r5D '../$(PKG_PATH)$(SYMBOL_FULL_ARCHIVE_BASENAME).zip' . -x '*test*' -x '*Test*'
 
 .PHONY: symbolsarchive
-symbolsarchive: generatesymbols
+symbolsarchive: prepsymbolsarchive
+	$(RM) '$(DIST)/$(SYMBOL_ARCHIVE_BASENAME).zip'
 	cd $(DIST)/crashreporter-symbols && \
           zip -r5D '../$(PKG_PATH)$(SYMBOL_ARCHIVE_BASENAME).zip' . -i '*.sym'
 
 ifdef MOZ_CRASHREPORTER
 buildsymbols: symbolsfullarchive symbolsarchive
 else
 buildsymbols:
 endif
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -875,16 +875,36 @@ crate_src_libdep = $(call mk_global_crat
 ifdef ASFILES
 # The AS_DASH_C_FLAG is needed cause not all assemblers (Solaris) accept
 # a '-c' flag.
 $(ASOBJS):
 	$(REPORT_BUILD_VERBOSE)
 	$(AS) $(ASOUTOPTION)$@ $(ASFLAGS) $($(notdir $<)_FLAGS) $(AS_DASH_C_FLAG) $(_VPATH_SRCS)
 endif
 
+define syms_template
+syms:: $(2)
+$(2): $(1)
+	$$(call py_action,dumpsymbols,$$(abspath $$<) $$(abspath $$@))
+endef
+
+ifndef MOZ_PROFILE_GENERATE
+ifneq (,$(filter $(DIST)/bin%,$(FINAL_TARGET)))
+DUMP_SYMS_TARGETS := $(SHARED_LIBRARY) $(PROGRAM) $(SIMPLE_PROGRAMS)
+endif
+endif
+
+ifdef MOZ_AUTOMATION
+ifeq (,$(filter 1,$(MOZ_AUTOMATION_BUILD_SYMBOLS)))
+DUMP_SYMS_TARGETS :=
+endif
+endif
+
+$(foreach file,$(DUMP_SYMS_TARGETS),$(eval $(call syms_template,$(file),$(file)_syms.track)))
+
 ifdef MOZ_RUST
 cargo_host_flag := --target=$(RUST_HOST_TARGET)
 cargo_target_flag := --target=$(RUST_TARGET)
 
 # Permit users to pass flags to cargo from their mozconfigs (e.g. --color=always).
 cargo_build_flags = $(CARGOFLAGS)
 ifndef MOZ_DEBUG
 cargo_build_flags += --release
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/action/dumpsymbols.py
@@ -0,0 +1,79 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import argparse
+import buildconfig
+import subprocess
+import shutil
+import sys
+import os
+
+def dump_symbols(target, tracking_file):
+    # Our tracking file, if present, will contain path(s) to the previously generated
+    # symbols. Remove them in this case so we don't simply accumulate old symbols
+    # during incremental builds.
+    if os.path.isfile(os.path.normpath(tracking_file)):
+        with open(tracking_file, 'r') as fh:
+            files = fh.read().splitlines()
+        dirs = set(os.path.dirname(f) for f in files)
+        for d in dirs:
+            shutil.rmtree(os.path.join(buildconfig.topobjdir, 'dist',
+                                       'crashreporter-symbols', d),
+                          ignore_errors=True)
+
+    # Build default args for symbolstore.py based on platform.
+    sym_store_args = []
+
+    dump_syms_bin = os.path.join(buildconfig.topobjdir,
+                                 'dist', 'host',
+                                 'bin', 'dump_syms')
+    dump_syms_bin = '%s%s' % (dump_syms_bin, buildconfig.substs['BIN_SUFFIX'])
+
+    os_arch = buildconfig.substs['OS_ARCH']
+    if os_arch == 'WINNT':
+        sym_store_args.extend(['-c', '--vcs-info'])
+        if os.environ.get('PDBSTR_PATH'):
+            sym_store_args.append('-i')
+    elif os_arch == 'Darwin':
+        sym_store_args.extend(['-c', '-a', buildconfig.substs['OS_TEST'], '--vcs-info'])
+    elif os_arch == 'Linux':
+        sym_store_args.extend(['-c', '--vcs-info'])
+
+    sym_store_args.append('--install-manifest=%s,%s' % (os.path.join(buildconfig.topobjdir,
+                                                                     '_build_manifests',
+                                                                     'install',
+                                                                     'dist_include'),
+                                                        os.path.join(buildconfig.topobjdir,
+                                                                     'dist',
+                                                                     'include')))
+    objcopy = buildconfig.substs.get('OBJCOPY')
+    if objcopy:
+        os.environ['OBJCOPY'] = objcopy
+
+    args = ([buildconfig.substs['PYTHON'], os.path.join(buildconfig.topsrcdir, 'toolkit',
+                                                       'crashreporter', 'tools', 'symbolstore.py')] +
+            sym_store_args +
+            ['-s', buildconfig.topsrcdir, dump_syms_bin, os.path.join(buildconfig.topobjdir,
+                                                                      'dist',
+                                                                      'crashreporter-symbols'),
+             os.path.abspath(target)])
+    print('Running: %s' % ' '.join(args))
+    out_files = subprocess.check_output(args)
+    with open(tracking_file, 'w') as fh:
+        fh.write(out_files)
+        fh.flush()
+
+def main(argv):
+    if len(argv) != 2:
+        print("Usage: dumpsymbols.py <library or program> <tracking file>",
+              file=sys.stderr)
+        return 1
+
+    return dump_symbols(*argv)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))
--- a/toolkit/crashreporter/tools/symbolstore.py
+++ b/toolkit/crashreporter/tools/symbolstore.py
@@ -572,17 +572,17 @@ class Dumper:
         for f in sorted(files, key=os.path.getsize, reverse=True):
             self.ProcessFiles((f,))
 
     def get_files_to_process(self, file_or_dir):
         """Generate the files to process from an input."""
         if os.path.isdir(file_or_dir) and not self.ShouldSkipDir(file_or_dir):
             for f in self.get_files_to_process_in_dir(file_or_dir):
                 yield f
-        elif os.path.isfile(file_or_dir):
+        elif os.path.isfile(file_or_dir) and self.ShouldProcess(os.path.abspath(file_or_dir)):
             yield file_or_dir
 
     def get_files_to_process_in_dir(self, path):
         """Generate the files to process in a directory.
 
         Valid files are are determined by calling ShouldProcess.
         """
         for root, dirs, files in os.walk(path):