bug 1255485 - build PROGRAMs directly in dist/bin instead of copying them. r=nalexander
authorTed Mielczarek <ted@mielczarek.org>
Wed, 10 Jan 2018 14:26:12 -0500
changeset 465001 ad09b92fb875944ba775ef452a245669b567619d
parent 465000 e2ae2186dbc91b9157dd810d1f3d15beddc73218
child 465002 6fffc6fbb8c9fab9b61c3fe54894e85e116f30df
push id1728
push userjlund@mozilla.com
push dateMon, 18 Jun 2018 21:12:27 +0000
treeherdermozilla-release@c296fde26f5f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander
bugs1255485
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
bug 1255485 - build PROGRAMs directly in dist/bin instead of copying them. r=nalexander Historically we built all our binaries in directories in the objdir, then symlinked them into dist/bin. Some binaries needed to be copied instead so that certain relative path lookups work properly, so we resorted to sprinkling `NSDISTMODE=copy` around Makefiles. This change makes it so we build PROGRAMs (not any other sort of targets) directly in dist/bin instead. We could do the same for our other targets with a little more work. There were several places in the tree that were copying built binaries to some other place and needed fixup to match the new location of binaries. On Windows pdb files are left in the objdir where the program was originally linked. symbolstore.py needs to locate the pdb file both to determine whether it should dump symbols for a binary and also to copy the pdb file into the symbol package. We fix this by simply looking for the pdb file in the current working directory if it isn't present next to the binary, which matches how we invoke symbolstore.py. MozReview-Commit-ID: 8TOD1uTXD5e
browser/app/Makefile.in
config/makefiles/target_binaries.mk
config/rules.mk
js/src/shell/moz.build
modules/libmar/tests/moz.build
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
toolkit/crashreporter/tools/symbolstore.py
toolkit/crashreporter/tools/unit-symbolstore.py
--- a/browser/app/Makefile.in
+++ b/browser/app/Makefile.in
@@ -41,17 +41,17 @@ endif
 PROGRAMS_DEST = $(DIST)/bin
 
 include $(topsrcdir)/config/rules.mk
 
 ifneq (,$(filter-out WINNT,$(OS_ARCH)))
 
 ifdef COMPILE_ENVIRONMENT
 libs::
-	cp -p $(MOZ_APP_NAME)$(BIN_SUFFIX) $(DIST)/bin/$(MOZ_APP_NAME)-bin$(BIN_SUFFIX)
+	cp -p $(DIST)/bin/$(MOZ_APP_NAME)$(BIN_SUFFIX) $(DIST)/bin/$(MOZ_APP_NAME)-bin$(BIN_SUFFIX)
 endif
 
 GARBAGE += $(addprefix $(FINAL_TARGET)/defaults/pref/, firefox.js)
 
 endif
 
 # channel-prefs.js is handled separate from other prefs due to bug 756325
 # DO NOT change the content of channel-prefs.js without taking the appropriate
--- a/config/makefiles/target_binaries.mk
+++ b/config/makefiles/target_binaries.mk
@@ -2,18 +2,18 @@
 # vim:set ts=8 sw=8 sts=8 noet:
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 ifndef NO_DIST_INSTALL
 
-ifneq (,$(strip $(PROGRAM)$(SIMPLE_PROGRAMS)$(RUST_PROGRAMS)))
-PROGRAMS_EXECUTABLES = $(SIMPLE_PROGRAMS) $(PROGRAM) $(RUST_PROGRAMS)
+ifneq (,$(strip $(SIMPLE_PROGRAMS)$(RUST_PROGRAMS)))
+PROGRAMS_EXECUTABLES = $(SIMPLE_PROGRAMS) $(RUST_PROGRAMS)
 PROGRAMS_DEST ?= $(FINAL_TARGET)
 PROGRAMS_TARGET := target
 INSTALL_TARGETS += PROGRAMS
 endif
 
 
 ifdef SHARED_LIBRARY
 SHARED_LIBRARY_FILES = $(SHARED_LIBRARY)
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -547,17 +547,17 @@ alltags:
 define EXPAND_CC_OR_CXX
 $(if $(PROG_IS_C_ONLY_$(1)),$(EXPAND_CC),$(EXPAND_CCC))
 endef
 
 #
 # PROGRAM = Foo
 # creates OBJS, links with LIBS to create Foo
 #
-$(PROGRAM): $(PROGOBJS) $(STATIC_LIBS_DEPS) $(EXTRA_DEPS) $(RESFILE) $(GLOBAL_DEPS)
+$(PROGRAM): $(PROGOBJS) $(STATIC_LIBS_DEPS) $(EXTRA_DEPS) $(RESFILE) $(GLOBAL_DEPS) $(call mkdir_deps,$(FINAL_TARGET))
 	$(REPORT_BUILD)
 	@$(RM) $@.manifest
 ifeq (_WINNT,$(GNU_CC)_$(OS_ARCH))
 	$(EXPAND_LINK) -NOLOGO -OUT:$@ -PDB:$(LINK_PDBFILE) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(MOZ_PROGRAM_LDFLAGS) $(PROGOBJS) $(RESFILE) $(STATIC_LIBS) $(SHARED_LIBS) $(OS_LIBS)
 ifdef MSMANIFEST_TOOL
 	@if test -f $@.manifest; then \
 		if test -f '$(srcdir)/$@.manifest'; then \
 			echo 'Embedding manifest from $(srcdir)/$@.manifest and $@.manifest'; \
@@ -812,17 +812,17 @@ endif
 
 ifdef MOZ_AUTOMATION
 ifeq (,$(filter 1,$(MOZ_AUTOMATION_BUILD_SYMBOLS)))
 DUMP_SYMS_TARGETS :=
 endif
 endif
 
 ifdef MOZ_CRASHREPORTER
-$(foreach file,$(DUMP_SYMS_TARGETS),$(eval $(call syms_template,$(file),$(file)_syms.track)))
+$(foreach file,$(DUMP_SYMS_TARGETS),$(eval $(call syms_template,$(file),$(notdir $(file))_syms.track)))
 else ifneq (,$(and $(LLVM_SYMBOLIZER),$(filter WINNT,$(OS_ARCH)),$(MOZ_AUTOMATION)))
 $(foreach file,$(DUMP_SYMS_TARGETS),$(eval $(call syms_template,$(file),$(addsuffix .pdb,$(basename $(file))))))
 PDB_FILES = $(addsuffix .pdb,$(basename $(DUMP_SYMS_TARGETS)))
 PDB_DEST ?= $(FINAL_TARGET)
 PDB_TARGET = target
 INSTALL_TARGETS += PDB
 endif
 
--- a/js/src/shell/moz.build
+++ b/js/src/shell/moz.build
@@ -59,9 +59,9 @@ if CONFIG['CC_TYPE'] in ('msvc', 'clang-
 
 # Place a GDB Python auto-load file next to the shell executable, both in
 # the build directory and in the dist/bin directory.
 DEFINES['topsrcdir'] = '%s/js/src' % TOPSRCDIR
 FINAL_TARGET_PP_FILES += ['js-gdb.py.in']
 OBJDIR_FILES.js.src.shell += ['!/dist/bin/js-gdb.py']
 
 # People expect the js shell to wind up in the top-level JS dir.
-OBJDIR_FILES.js.src += ['!js%s' % CONFIG['BIN_SUFFIX']]
+OBJDIR_FILES.js.src += ['!/dist/bin/js%s' % CONFIG['BIN_SUFFIX']]
--- a/modules/libmar/tests/moz.build
+++ b/modules/libmar/tests/moz.build
@@ -3,10 +3,10 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini']
 
 if CONFIG['OS_TARGET'] != 'Android':
     TEST_HARNESS_FILES.xpcshell.modules.libmar.tests.unit += [
-        '!../tool/signmar%s' % CONFIG['BIN_SUFFIX'],
+        '!/dist/bin/signmar%s' % CONFIG['BIN_SUFFIX'],
     ]
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -1124,17 +1124,17 @@ class RecursiveMakeBackend(CommonBackend
             interfaces_manifests = ' '.join(interfaces_manifests),
             xpidl_rules=rules.getvalue(),
             xpidl_modules=' '.join(xpt_modules),
             xpt_files=' '.join(sorted(xpt_files - registered_xpt_files)),
             registered_xpt_files=' '.join(sorted(registered_xpt_files)),
         ))
 
     def _process_program(self, obj, backend_file):
-        backend_file.write('PROGRAM = %s\n' % obj.program)
+        backend_file.write('PROGRAM = %s\n' % self._pretty_path(obj.output_path, backend_file))
         if not obj.cxx_link and not self.environment.bin_suffix:
             backend_file.write('PROG_IS_C_ONLY_%s := 1\n' % obj.program)
 
     def _process_host_program(self, program, backend_file):
         backend_file.write('HOST_PROGRAM = %s\n' % program)
 
     def _process_rust_program_base(self, obj, backend_file,
                                    target_variable,
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -12,16 +12,17 @@ All data structures of interest are chil
 Logic for populating these data structures is not defined in this class.
 Instead, what we have here are dumb container classes. The emitter module
 contains the code for converting executed mozbuild files into these data
 structures.
 """
 
 from __future__ import absolute_import, unicode_literals
 
+from mozbuild.frontend.context import ObjDirPath
 from mozbuild.util import StrictOrderingOnAppendList
 from mozpack.chrome.manifest import ManifestEntry
 
 import mozpack.path as mozpath
 from .context import FinalTargetValue
 
 from collections import defaultdict, OrderedDict
 import itertools
@@ -81,16 +82,20 @@ class ContextDerived(TreeMetadata):
 
         self._context = context
 
     @property
     def install_target(self):
         return self._context['FINAL_TARGET']
 
     @property
+    def installed(self):
+        return self._context['DIST_INSTALL'] is not False
+
+    @property
     def defines(self):
         defines = self._context['DEFINES']
         return Defines(self._context, defines) if defines else None
 
     @property
     def relobjdir(self):
         return mozpath.relpath(self.objdir, self.topobjdir)
 
@@ -467,16 +472,23 @@ class BaseProgram(Linkable):
         Linkable.__init__(self, context)
 
         bin_suffix = context.config.substs.get(self.SUFFIX_VAR, '')
         if not program.endswith(bin_suffix):
             program += bin_suffix
         self.program = program
         self.is_unit_test = is_unit_test
 
+    @property
+    def output_path(self):
+        if self.installed:
+            return ObjDirPath(self._context, '!/' + mozpath.join(self.install_target, self.program))
+        else:
+            return ObjDirPath(self._context, '!' + self.program)
+
     def __repr__(self):
         return '<%s: %s/%s>' % (type(self).__name__, self.relobjdir, self.program)
 
 
 class Program(BaseProgram):
     """Context derived container object for PROGRAM"""
     SUFFIX_VAR = 'BIN_SUFFIX'
     KIND = 'target'
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -779,17 +779,18 @@ class TreeMetadataEmitter(LoggingMixin):
                              'GENERATED_FILES: %s') % (symbols_file,), context)
                     shared_args['symbols_file'] = symbols_file.target_basename
 
             if shared_lib:
                 lib = SharedLibrary(context, libname, **shared_args)
                 self._libs[libname].append(lib)
                 self._linkage.append((context, lib, 'USE_LIBS'))
                 linkables.append(lib)
-                generated_files.add(lib.lib_name)
+                if not lib.installed:
+                    generated_files.add(lib.lib_name)
                 if symbols_file and isinstance(symbols_file, SourcePath):
                     script = mozpath.join(
                         mozpath.dirname(mozpath.dirname(__file__)),
                         'action', 'generate_symbols_file.py')
                     defines = ()
                     if lib.defines:
                         defines = lib.defines.get_defines()
                     yield GeneratedFile(context, script,
--- a/toolkit/crashreporter/tools/symbolstore.py
+++ b/toolkit/crashreporter/tools/symbolstore.py
@@ -607,31 +607,50 @@ class Dumper:
 
         elapsed = time.time() - t_start
         print('Finished processing %s in %.2fs' % (file, elapsed),
               file=sys.stderr)
 
 # Platform-specific subclasses.  For the most part, these just have
 # logic to determine what files to extract symbols from.
 
+def locate_pdb(path):
+    '''Given a path to a binary, attempt to locate the matching pdb file with simple heuristics:
+    * Look for a pdb file with the same base name next to the binary
+    * Look for a pdb file with the same base name in the cwd
+
+    Returns the path to the pdb file if it exists, or None if it could not be located.
+    '''
+    path, ext = os.path.splitext(path)
+    pdb = path + '.pdb'
+    if os.path.isfile(pdb):
+        return pdb
+    # If there's no pdb next to the file, see if there's a pdb with the same root name
+    # in the cwd. We build some binaries directly into dist/bin, but put the pdb files
+    # in the relative objdir, which is the cwd when running this script.
+    base = os.path.basename(pdb)
+    pdb = os.path.join(os.getcwd(), base)
+    if os.path.isfile(pdb):
+        return pdb
+    return None
+
 class Dumper_Win32(Dumper):
     fixedFilenameCaseCache = {}
 
     def ShouldProcess(self, file):
         """This function will allow processing of exe or dll files that have pdb
         files with the same base name next to them."""
         if file.endswith(".exe") or file.endswith(".dll"):
-            path, ext = os.path.splitext(file)
-            if os.path.isfile(path + ".pdb"):
+            if locate_pdb(file) is not None:
                 return True
         return False
 
 
     def CopyDebug(self, file, debug_file, guid, code_file, code_id):
-        file = "%s.pdb" % os.path.splitext(file)[0]
+        file = locate_pdb(file)
         def compress(path):
             compressed_file = path[:-1] + '_'
             # ignore makecab's output
             makecab = buildconfig.substs['MAKECAB']
             success = subprocess.call([makecab, "-D",
                                        "CompressionType=MSZIP",
                                        path, compressed_file],
                                       stdout=open(os.devnull, 'w'),
--- a/toolkit/crashreporter/tools/unit-symbolstore.py
+++ b/toolkit/crashreporter/tools/unit-symbolstore.py
@@ -479,18 +479,17 @@ class TestFunctional(HelperMixin, unitte
             else:
                 self.dump_syms = os.path.join(self.topsrcdir,
                                               'toolkit',
                                               'crashreporter',
                                               'tools',
                                               'win32',
                                               'dump_syms_vc{_MSC_VER}.exe'.format(**buildconfig.substs))
             self.target_bin = os.path.join(buildconfig.topobjdir,
-                                           'browser',
-                                           'app',
+                                           'dist', 'bin',
                                            'firefox.exe')
         else:
             self.dump_syms = os.path.join(buildconfig.topobjdir,
                                           'dist', 'host', 'bin',
                                           'dump_syms')
             self.target_bin = os.path.join(buildconfig.topobjdir,
                                            'dist', 'bin', 'firefox')
 
@@ -499,26 +498,28 @@ class TestFunctional(HelperMixin, unitte
         HelperMixin.tearDown(self)
 
     def testSymbolstore(self):
         if self.skip_test:
             raise unittest.SkipTest('Skipping test in non-Firefox product')
         dist_include_manifest = os.path.join(buildconfig.topobjdir,
                                              '_build_manifests/install/dist_include')
         dist_include = os.path.join(buildconfig.topobjdir, 'dist/include')
+        browser_app = os.path.join(buildconfig.topobjdir, 'browser/app')
         output = subprocess.check_output([sys.executable,
                                           self.script_path,
                                           '--vcs-info',
                                           '-s', self.topsrcdir,
                                           '--install-manifest=%s,%s' % (dist_include_manifest,
                                                                         dist_include),
                                           self.dump_syms,
                                           self.test_dir,
                                           self.target_bin],
-                                         stderr=open(os.devnull, 'w'))
+                                         stderr=open(os.devnull, 'w'),
+                                         cwd=browser_app)
         lines = filter(lambda x: x.strip(), output.splitlines())
         self.assertEqual(1, len(lines),
                          'should have one filename in the output')
         symbol_file = os.path.join(self.test_dir, lines[0])
         self.assertTrue(os.path.isfile(symbol_file))
         symlines = open(symbol_file, 'r').readlines()
         file_lines = [l for l in symlines if l.startswith('FILE')]
         def check_hg_path(lines, match):