Bug 1429875 - Implement OBJ_SUFFIX overriding for the profile generation phase on linux in mozbuild. draft
authorChris Manchester <cmanchester@mozilla.com>
Tue, 20 Mar 2018 16:44:12 -0700
changeset 777658 6839d78c3d8dd90489b54c66ba691953bc2de102
parent 777657 a013b0f740db43b89573b0245f9b401728a69b74
child 777659 e74191e0592073d8f275c51e59f298000967efc2
push id105261
push userbmo:tom@mozilla.com
push dateThu, 05 Apr 2018 04:18:27 +0000
bugs1429875
milestone61.0a1
Bug 1429875 - Implement OBJ_SUFFIX overriding for the profile generation phase on linux in mozbuild. MozReview-Commit-ID: 8PtgxfbxuE
config/config.mk
config/rules.mk
python/mozbuild/mozbuild/backend/common.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
--- a/config/config.mk
+++ b/config/config.mk
@@ -469,25 +469,28 @@ define CHECK_BINARY
 $(call LOCAL_CHECKS,$(1))
 $(call CHECK_MOZGLUE_ORDER,$(1))
 endef
 
 # autoconf.mk sets OBJ_SUFFIX to an error to avoid use before including
 # this file
 OBJ_SUFFIX := $(_OBJ_SUFFIX)
 
+OBJS_VAR_SUFFIX := OBJS
+
 # PGO builds with GCC build objects with instrumentation in a first pass,
 # then objects optimized, without instrumentation, in a second pass. If
 # we overwrite the objects from the first pass with those from the second,
 # we end up not getting instrumentation data for better optimization on
 # incremental builds. As a consequence, we use a different object suffix
 # for the first pass.
-ifndef NO_PROFILE_GUIDED_OPTIMIZE
 ifdef MOZ_PROFILE_GENERATE
 ifdef GNU_CC
+OBJS_VAR_SUFFIX := PGO_OBJS
+ifndef NO_PROFILE_GUIDED_OPTIMIZE
 OBJ_SUFFIX := i_o
 endif
 endif
 endif
 
 PLY_INCLUDE = -I$(MOZILLA_DIR)/other-licenses/ply
 
 export CL_INCLUDES_PREFIX
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -538,17 +538,17 @@ endef
 #
 # PROGRAM = Foo
 # creates OBJS, links with LIBS to create Foo
 #
 $(PROGRAM): $(PROGOBJS) $(STATIC_LIBS) $(EXTRA_DEPS) $(RESFILE) $(GLOBAL_DEPS) $(call mkdir_deps,$(FINAL_TARGET))
 	$(REPORT_BUILD)
 	@$(RM) $@.manifest
 ifeq (_WINNT,$(GNU_CC)_$(OS_ARCH))
-	$(LINKER) -NOLOGO -OUT:$@ -PDB:$(LINK_PDBFILE) -IMPLIB:$(basename $(@F)).lib $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(MOZ_PROGRAM_LDFLAGS) $($(notdir $@)_OBJS) $(RESFILE) $(STATIC_LIBS) $(SHARED_LIBS) $(OS_LIBS)
+	$(LINKER) -NOLOGO -OUT:$@ -PDB:$(LINK_PDBFILE) -IMPLIB:$(basename $(@F)).lib $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(MOZ_PROGRAM_LDFLAGS) $($(notdir $@)_$(OBJS_VAR_SUFFIX)) $(RESFILE) $(STATIC_LIBS) $(SHARED_LIBS) $(OS_LIBS)
 ifdef MSMANIFEST_TOOL
 	@if test -f $@.manifest; then \
 		if test -f '$(srcdir)/$(notdir $@).manifest'; then \
 			echo 'Embedding manifest from $(srcdir)/$(notdir $@).manifest and $@.manifest'; \
 			$(MT) -NOLOGO -MANIFEST '$(win_srcdir)/$(notdir $@).manifest' $@.manifest -OUTPUTRESOURCE:$@\;1; \
 		else \
 			echo 'Embedding manifest from $@.manifest'; \
 			$(MT) -NOLOGO -MANIFEST $@.manifest -OUTPUTRESOURCE:$@\;1; \
@@ -559,17 +559,17 @@ ifdef MSMANIFEST_TOOL
 	fi
 endif	# MSVC with manifest tool
 ifdef MOZ_PROFILE_GENERATE
 # touch it a few seconds into the future to work around FAT's
 # 2-second granularity
 	touch -t `date +%Y%m%d%H%M.%S -d 'now+5seconds'` pgo.relink
 endif
 else # !WINNT || GNU_CC
-	$(call EXPAND_CC_OR_CXX,$@) -o $@ $(COMPUTED_CXX_LDFLAGS) $(PGO_CFLAGS) $($(notdir $@)_OBJS) $(RESFILE) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(STATIC_LIBS) $(MOZ_PROGRAM_LDFLAGS) $(SHARED_LIBS) $(OS_LIBS)
+	$(call EXPAND_CC_OR_CXX,$@) -o $@ $(COMPUTED_CXX_LDFLAGS) $(PGO_CFLAGS) $($(notdir $@)_$(OBJS_VAR_SUFFIX)) $(RESFILE) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(STATIC_LIBS) $(MOZ_PROGRAM_LDFLAGS) $(SHARED_LIBS) $(OS_LIBS)
 	$(call CHECK_BINARY,$@)
 endif # WINNT && !GNU_CC
 
 ifdef ENABLE_STRIP
 	$(STRIP) $(STRIP_FLAGS) $@
 endif
 ifdef MOZ_POST_PROGRAM_COMMAND
 	$(MOZ_POST_PROGRAM_COMMAND) $@
@@ -610,25 +610,25 @@ endif
 # Foo.o (from either Foo.c or Foo.cpp).
 #
 # SIMPLE_PROGRAMS = Foo Bar
 # creates Foo.o Bar.o, links with LIBS to create Foo, Bar.
 #
 $(SIMPLE_PROGRAMS): %$(BIN_SUFFIX): %.$(OBJ_SUFFIX) $(STATIC_LIBS) $(EXTRA_DEPS) $(GLOBAL_DEPS)
 	$(REPORT_BUILD)
 ifeq (_WINNT,$(GNU_CC)_$(OS_ARCH))
-	$(LINKER) -nologo -out:$@ -pdb:$(LINK_PDBFILE) $($@_OBJS) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(MOZ_PROGRAM_LDFLAGS) $(STATIC_LIBS) $(SHARED_LIBS) $(OS_LIBS)
+	$(LINKER) -nologo -out:$@ -pdb:$(LINK_PDBFILE) $($@_$(OBJS_VAR_SUFFIX)) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(MOZ_PROGRAM_LDFLAGS) $(STATIC_LIBS) $(SHARED_LIBS) $(OS_LIBS)
 ifdef MSMANIFEST_TOOL
 	@if test -f $@.manifest; then \
 		$(MT) -NOLOGO -MANIFEST $@.manifest -OUTPUTRESOURCE:$@\;1; \
 		rm -f $@.manifest; \
 	fi
 endif	# MSVC with manifest tool
 else
-	$(call EXPAND_CC_OR_CXX,$@) $(COMPUTED_CXX_LDFLAGS) $(PGO_CFLAGS) -o $@ $($@_OBJS) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(STATIC_LIBS) $(MOZ_PROGRAM_LDFLAGS) $(SHARED_LIBS) $(OS_LIBS)
+	$(call EXPAND_CC_OR_CXX,$@) $(COMPUTED_CXX_LDFLAGS) $(PGO_CFLAGS) -o $@ $($@_$(OBJS_VAR_SUFFIX)) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(STATIC_LIBS) $(MOZ_PROGRAM_LDFLAGS) $(SHARED_LIBS) $(OS_LIBS)
 	$(call CHECK_BINARY,$@)
 endif # WINNT && !GNU_CC
 
 ifdef ENABLE_STRIP
 	$(STRIP) $(STRIP_FLAGS) $@
 endif
 ifdef MOZ_POST_PROGRAM_COMMAND
 	$(MOZ_POST_PROGRAM_COMMAND) $@
@@ -647,17 +647,17 @@ endif
 endif
 ifndef CROSS_COMPILE
 	$(call CHECK_STDCXX,$@)
 endif
 
 $(LIBRARY): $(OBJS) $(STATIC_LIBS) $(EXTRA_DEPS) $(GLOBAL_DEPS)
 	$(REPORT_BUILD)
 	$(RM) $(REAL_LIBRARY)
-	$(AR) $(AR_FLAGS) $(OBJS) $($@_OBJS)
+	$(AR) $(AR_FLAGS) $(OBJS) $($@_$(OBJS_VAR_SUFFIX))
 
 ifeq ($(OS_ARCH),WINNT)
 # Import libraries are created by the rules creating shared libraries.
 # The rules to copy them to $(DIST)/lib depend on $(IMPORT_LIBRARY),
 # but make will happily consider the import library before it is refreshed
 # when rebuilding the corresponding shared library. Defining an empty recipe
 # for import libraries forces make to wait for the shared library recipe to
 # have run before considering other targets that depend on the import library.
@@ -675,17 +675,17 @@ endif
 # symlinks back to the originals. The symlinks are a no-op for stabs debugging,
 # so no need to conditionalize on OS version or debugging format.
 
 $(SHARED_LIBRARY): $(OBJS) $(RESFILE) $(RUST_STATIC_LIB_FOR_SHARED_LIB) $(STATIC_LIBS) $(EXTRA_DEPS) $(GLOBAL_DEPS)
 	$(REPORT_BUILD)
 ifndef INCREMENTAL_LINKER
 	$(RM) $@
 endif
-	$(MKSHLIB) $($@_OBJS) $(RESFILE) $(LDFLAGS) $(STATIC_LIBS) $(RUST_STATIC_LIB_FOR_SHARED_LIB) $(SHARED_LIBS) $(EXTRA_DSO_LDOPTS) $(MOZ_GLUE_LDFLAGS) $(OS_LIBS)
+	$(MKSHLIB) $($@_$(OBJS_VAR_SUFFIX)) $(RESFILE) $(LDFLAGS) $(STATIC_LIBS) $(RUST_STATIC_LIB_FOR_SHARED_LIB) $(SHARED_LIBS) $(EXTRA_DSO_LDOPTS) $(MOZ_GLUE_LDFLAGS) $(OS_LIBS)
 	$(call CHECK_BINARY,$@)
 
 ifeq (_WINNT,$(GNU_CC)_$(OS_ARCH))
 ifdef MSMANIFEST_TOOL
 ifdef EMBED_MANIFEST_AT
 	@if test -f $@.manifest; then \
 		if test -f '$(srcdir)/$@.manifest'; then \
 			echo 'Embedding manifest from $(srcdir)/$@.manifest and $@.manifest'; \
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -203,25 +203,31 @@ class CommonBackend(BuildBackend):
             }
             json.dump(d, fh, sort_keys=True, indent=4)
 
     def _expand_libs(self, input_bin):
         os_libs = []
         shared_libs = []
         static_libs = []
         objs = []
+        no_pgo_objs = []
 
         seen_objs = set()
         seen_libs = set()
 
         def add_objs(lib):
             for o in lib.objs:
                 if o not in seen_objs:
                     seen_objs.add(o)
                     objs.append(o)
+                    # This is slightly odd, buf for consistency with the
+                    # recursivemake backend we don't replace OBJ_SUFFIX if any
+                    # object in a library has `no_pgo` set.
+                    if lib.no_pgo_objs or lib.no_pgo:
+                        no_pgo_objs.append(o)
 
         def expand(lib, recurse_objs, system_libs):
             if isinstance(lib, StaticLibrary):
                 if lib.no_expand_lib:
                     static_libs.append(lib)
                     recurse_objs = False
                 elif recurse_objs:
                     add_objs(lib)
@@ -253,17 +259,17 @@ class CommonBackend(BuildBackend):
                     seen_libs.add(lib)
                     shared_libs.append(lib)
 
         for lib in input_bin.linked_system_libs:
             if lib not in seen_libs:
                 seen_libs.add(lib)
                 os_libs.append(lib)
 
-        return objs, shared_libs, os_libs, static_libs
+        return objs, no_pgo_objs, shared_libs, os_libs, static_libs
 
     def _make_list_file(self, objdir, objs, name):
         if not objs:
             return None
         list_style = self.environment.substs.get('EXPAND_LIBS_LIST_STYLE')
         list_file_path = mozpath.join(objdir, name)
         objs = [os.path.relpath(o, objdir) for o in objs]
         if list_style == 'linkerscript':
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -1320,43 +1320,78 @@ class RecursiveMakeBackend(CommonBackend
         def pretty_relpath(lib):
             return '$(DEPTH)/%s' % mozpath.relpath(lib.objdir, topobjdir)
 
         topobjdir = mozpath.normsep(obj.topobjdir)
         # This will create the node even if there aren't any linked libraries.
         build_target = self._build_target_for_obj(obj)
         self._compile_graph[build_target]
 
-        objs, shared_libs, os_libs, static_libs = self._expand_libs(obj)
+        objs, no_pgo_objs, shared_libs, os_libs, static_libs = self._expand_libs(obj)
 
         if obj.KIND == 'target':
             obj_target = obj.name
             if isinstance(obj, Program):
                 obj_target = self._pretty_path(obj.output_path, backend_file)
 
+            is_unit_test = isinstance(obj, BaseProgram) and obj.is_unit_test
+            profile_gen_objs = []
+
+            if (self.environment.substs.get('MOZ_PGO') and
+                self.environment.substs.get('GNU_CC')):
+                # We use a different OBJ_SUFFIX for the profile generate phase on
+                # linux. These get picked up via OBJS_VAR_SUFFIX in config.mk.
+                if not is_unit_test and not isinstance(obj, SimpleProgram):
+                    profile_gen_objs = [o if o in no_pgo_objs else '%s.%s' %
+                                        (mozpath.splitext(o)[0], 'i_o') for o in objs]
+
+            def write_obj_deps(target, objs_ref, pgo_objs_ref):
+                if pgo_objs_ref:
+                    backend_file.write('ifdef MOZ_PROFILE_GENERATE\n')
+                    backend_file.write('%s: %s\n' % (target, pgo_objs_ref))
+                    backend_file.write('else\n')
+                    backend_file.write('%s: %s\n' % (target, objs_ref))
+                    backend_file.write('endif\n')
+                else:
+                    backend_file.write('%s: %s\n' % (target, objs_ref))
+
             objs_ref = ' \\\n    '.join(os.path.relpath(o, obj.objdir)
                                         for o in objs)
+            pgo_objs_ref = ' \\\n    '.join(os.path.relpath(o, obj.objdir)
+                                            for o in profile_gen_objs)
             # Don't bother with a list file if we're only linking objects built
             # in this directory or building a real static library. This
             # accommodates clang-plugin, where we would otherwise pass an
             # incorrect list file format to the host compiler as well as when
             # creating an archive with AR, which doesn't understand list files.
             if (objs == obj.objs and not isinstance(obj, StaticLibrary) or
-                isinstance(obj, StaticLibrary) and obj.no_expand_lib):
-                backend_file.write_once('%s_OBJS := %s\n' %
-                                        (obj.name, objs_ref))
-                backend_file.write_once('%s: %s\n' % (obj_target, objs_ref))
+              isinstance(obj, StaticLibrary) and obj.no_expand_lib):
+                backend_file.write_once('%s_OBJS := %s\n' % (obj.name,
+                                                             objs_ref))
+                if profile_gen_objs:
+                    backend_file.write_once('%s_PGO_OBJS := %s\n' % (obj.name,
+                                                                     pgo_objs_ref))
+                write_obj_deps(obj_target, objs_ref, pgo_objs_ref)
             elif not isinstance(obj, StaticLibrary):
                 list_file_path = '%s.list' % obj.name.replace('.', '_')
                 list_file_ref = self._make_list_file(obj.objdir, objs,
                                                      list_file_path)
                 backend_file.write_once('%s_OBJS := %s\n' %
                                         (obj.name, list_file_ref))
                 backend_file.write_once('%s: %s\n' % (obj_target, list_file_path))
-                backend_file.write_once('%s: %s\n' % (obj_target, objs_ref))
+                if profile_gen_objs:
+                    pgo_list_file_path = '%s_pgo.list' % obj.name.replace('.', '_')
+                    pgo_list_file_ref = self._make_list_file(obj.objdir,
+                                                             profile_gen_objs,
+                                                             pgo_list_file_path)
+                    backend_file.write_once('%s_PGO_OBJS := %s\n' %
+                                            (obj.name, pgo_list_file_ref))
+                    backend_file.write_once('%s: %s\n' % (obj_target,
+                                                          pgo_list_file_path))
+                write_obj_deps(obj_target, objs_ref, pgo_objs_ref)
 
         for lib in shared_libs:
             backend_file.write_once('SHARED_LIBS += %s/%s\n' %
                                     (pretty_relpath(lib), lib.import_name))
         for lib in static_libs:
             backend_file.write_once('STATIC_LIBS += %s/%s\n' %
                                     (pretty_relpath(lib), lib.import_name))
         for lib in os_libs:
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -383,26 +383,30 @@ class LinkageMultipleRustLibrariesError(
 
 class Linkable(ContextDerived):
     """Generic context derived container object for programs and libraries"""
     __slots__ = (
         'cxx_link',
         'lib_defines',
         'linked_libraries',
         'linked_system_libs',
+        'no_pgo_sources',
+        'no_pgo',
         'sources',
     )
 
     def __init__(self, context):
         ContextDerived.__init__(self, context)
         self.cxx_link = False
         self.linked_libraries = []
         self.linked_system_libs = []
         self.lib_defines = Defines(context, {})
         self.sources = defaultdict(list)
+        self.no_pgo_sources = []
+        self.no_pgo = False
 
     def link_library(self, obj):
         assert isinstance(obj, BaseLibrary)
         if obj.KIND != self.KIND:
             raise LinkageWrongKindError('%s != %s' % (obj.KIND, self.KIND))
         # Linking multiple Rust libraries into an object would result in
         # multiple copies of the Rust standard library, as well as linking
         # errors from duplicate symbols.
@@ -432,26 +436,33 @@ class Linkable(ContextDerived):
     def source_files(self):
         all_sources = []
         # This is ordered for reproducibility and consistently w/
         # config/rules.mk
         for suffix in ('.c', '.S', '.cpp', '.m', '.mm', '.s'):
             all_sources += self.sources.get(suffix, [])
         return all_sources
 
-    @property
-    def objs(self):
+    def _get_objs(self, sources):
         obj_prefix = ''
         if self.KIND == 'host':
             obj_prefix = 'host_'
 
         return [mozpath.join(self.objdir, '%s%s.%s' % (obj_prefix,
                                                        mozpath.splitext(mozpath.basename(f))[0],
                                                        self.config.substs.get('OBJ_SUFFIX', '')))
-                for f in self.source_files()]
+                for f in sources]
+
+    @property
+    def no_pgo_objs(self):
+        return self._get_objs(self.no_pgo_sources)
+
+    @property
+    def objs(self):
+        return self._get_objs(self.source_files())
 
 
 class BaseProgram(Linkable):
     """Context derived container object for programs, which is a unicode
     string.
 
     This class handles automatically appending a binary suffix to the program
     name.
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -946,16 +946,20 @@ class TreeMetadataEmitter(LoggingMixin):
                     ctxt_sources[variable][canonical_suffix] += sorted(srcs)
                     yield obj
 
         if ctxt_sources:
             for linkable in linkables:
                 for target_var in ('SOURCES', 'UNIFIED_SOURCES'):
                     for suffix, srcs in ctxt_sources[target_var].items():
                         linkable.sources[suffix] += srcs
+                if no_pgo_sources:
+                    linkable.no_pgo_sources = no_pgo_sources
+                elif no_pgo:
+                    linkable.no_pgo = True
             for host_linkable in host_linkables:
                 for suffix, srcs in ctxt_sources['HOST_SOURCES'].items():
                     host_linkable.sources[suffix] += srcs
 
         for f, flags in all_flags.iteritems():
             if flags.flags:
                 ext = mozpath.splitext(f)[1]
                 yield PerSourceFlag(context, f, flags.flags)