Bug 1429875 - Implement OBJ_SUFFIX overriding for the profile generation phase on linux in mozbuild. r=glandium
authorChris Manchester <cmanchester@mozilla.com>
Tue, 20 Mar 2018 16:44:12 -0700
changeset 409254 cd90d8ccc5be982afb1bab0a2f2a649e10491817
parent 409253 7547d66e0f518d20f61b492eac027d5a34ac6fb4
child 409255 bac1aef09474835edae8d59c59bc44aa2e2270c4
push id61478
push usercmanchester@mozilla.com
push dateWed, 21 Mar 2018 18:02:15 +0000
treeherderautoland@cd90d8ccc5be [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1429875
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1429875 - Implement OBJ_SUFFIX overriding for the profile generation phase on linux in mozbuild. r=glandium MozReview-Commit-ID: 8PtgxfbxuE
config/config.mk
config/rules.mk
python/mozbuild/mozbuild/backend/common.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
--- a/config/config.mk
+++ b/config/config.mk
@@ -469,25 +469,28 @@ define CHECK_BINARY
 $(call LOCAL_CHECKS,$(1))
 $(call CHECK_MOZGLUE_ORDER,$(1))
 endef
 
 # autoconf.mk sets OBJ_SUFFIX to an error to avoid use before including
 # this file
 OBJ_SUFFIX := $(_OBJ_SUFFIX)
 
+OBJS_VAR_SUFFIX := OBJS
+
 # PGO builds with GCC build objects with instrumentation in a first pass,
 # then objects optimized, without instrumentation, in a second pass. If
 # we overwrite the objects from the first pass with those from the second,
 # we end up not getting instrumentation data for better optimization on
 # incremental builds. As a consequence, we use a different object suffix
 # for the first pass.
-ifndef NO_PROFILE_GUIDED_OPTIMIZE
 ifdef MOZ_PROFILE_GENERATE
 ifdef GNU_CC
+OBJS_VAR_SUFFIX := PGO_OBJS
+ifndef NO_PROFILE_GUIDED_OPTIMIZE
 OBJ_SUFFIX := i_o
 endif
 endif
 endif
 
 PLY_INCLUDE = -I$(MOZILLA_DIR)/other-licenses/ply
 
 export CL_INCLUDES_PREFIX
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -538,17 +538,17 @@ endef
 #
 # PROGRAM = Foo
 # creates OBJS, links with LIBS to create Foo
 #
 $(PROGRAM): $(PROGOBJS) $(STATIC_LIBS) $(EXTRA_DEPS) $(RESFILE) $(GLOBAL_DEPS) $(call mkdir_deps,$(FINAL_TARGET))
 	$(REPORT_BUILD)
 	@$(RM) $@.manifest
 ifeq (_WINNT,$(GNU_CC)_$(OS_ARCH))
-	$(LINKER) -NOLOGO -OUT:$@ -PDB:$(LINK_PDBFILE) -IMPLIB:$(basename $(@F)).lib $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(MOZ_PROGRAM_LDFLAGS) $($(notdir $@)_OBJS) $(RESFILE) $(STATIC_LIBS) $(SHARED_LIBS) $(OS_LIBS)
+	$(LINKER) -NOLOGO -OUT:$@ -PDB:$(LINK_PDBFILE) -IMPLIB:$(basename $(@F)).lib $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(MOZ_PROGRAM_LDFLAGS) $($(notdir $@)_$(OBJS_VAR_SUFFIX)) $(RESFILE) $(STATIC_LIBS) $(SHARED_LIBS) $(OS_LIBS)
 ifdef MSMANIFEST_TOOL
 	@if test -f $@.manifest; then \
 		if test -f '$(srcdir)/$(notdir $@).manifest'; then \
 			echo 'Embedding manifest from $(srcdir)/$(notdir $@).manifest and $@.manifest'; \
 			$(MT) -NOLOGO -MANIFEST '$(win_srcdir)/$(notdir $@).manifest' $@.manifest -OUTPUTRESOURCE:$@\;1; \
 		else \
 			echo 'Embedding manifest from $@.manifest'; \
 			$(MT) -NOLOGO -MANIFEST $@.manifest -OUTPUTRESOURCE:$@\;1; \
@@ -559,17 +559,17 @@ ifdef MSMANIFEST_TOOL
 	fi
 endif	# MSVC with manifest tool
 ifdef MOZ_PROFILE_GENERATE
 # touch it a few seconds into the future to work around FAT's
 # 2-second granularity
 	touch -t `date +%Y%m%d%H%M.%S -d 'now+5seconds'` pgo.relink
 endif
 else # !WINNT || GNU_CC
-	$(call EXPAND_CC_OR_CXX,$@) -o $@ $(COMPUTED_CXX_LDFLAGS) $(PGO_CFLAGS) $($(notdir $@)_OBJS) $(RESFILE) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(STATIC_LIBS) $(MOZ_PROGRAM_LDFLAGS) $(SHARED_LIBS) $(OS_LIBS)
+	$(call EXPAND_CC_OR_CXX,$@) -o $@ $(COMPUTED_CXX_LDFLAGS) $(PGO_CFLAGS) $($(notdir $@)_$(OBJS_VAR_SUFFIX)) $(RESFILE) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(STATIC_LIBS) $(MOZ_PROGRAM_LDFLAGS) $(SHARED_LIBS) $(OS_LIBS)
 	$(call CHECK_BINARY,$@)
 endif # WINNT && !GNU_CC
 
 ifdef ENABLE_STRIP
 	$(STRIP) $(STRIP_FLAGS) $@
 endif
 ifdef MOZ_POST_PROGRAM_COMMAND
 	$(MOZ_POST_PROGRAM_COMMAND) $@
@@ -610,25 +610,25 @@ endif
 # Foo.o (from either Foo.c or Foo.cpp).
 #
 # SIMPLE_PROGRAMS = Foo Bar
 # creates Foo.o Bar.o, links with LIBS to create Foo, Bar.
 #
 $(SIMPLE_PROGRAMS): %$(BIN_SUFFIX): %.$(OBJ_SUFFIX) $(STATIC_LIBS) $(EXTRA_DEPS) $(GLOBAL_DEPS)
 	$(REPORT_BUILD)
 ifeq (_WINNT,$(GNU_CC)_$(OS_ARCH))
-	$(LINKER) -nologo -out:$@ -pdb:$(LINK_PDBFILE) $($@_OBJS) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(MOZ_PROGRAM_LDFLAGS) $(STATIC_LIBS) $(SHARED_LIBS) $(OS_LIBS)
+	$(LINKER) -nologo -out:$@ -pdb:$(LINK_PDBFILE) $($@_$(OBJS_VAR_SUFFIX)) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(MOZ_PROGRAM_LDFLAGS) $(STATIC_LIBS) $(SHARED_LIBS) $(OS_LIBS)
 ifdef MSMANIFEST_TOOL
 	@if test -f $@.manifest; then \
 		$(MT) -NOLOGO -MANIFEST $@.manifest -OUTPUTRESOURCE:$@\;1; \
 		rm -f $@.manifest; \
 	fi
 endif	# MSVC with manifest tool
 else
-	$(call EXPAND_CC_OR_CXX,$@) $(COMPUTED_CXX_LDFLAGS) $(PGO_CFLAGS) -o $@ $($@_OBJS) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(STATIC_LIBS) $(MOZ_PROGRAM_LDFLAGS) $(SHARED_LIBS) $(OS_LIBS)
+	$(call EXPAND_CC_OR_CXX,$@) $(COMPUTED_CXX_LDFLAGS) $(PGO_CFLAGS) -o $@ $($@_$(OBJS_VAR_SUFFIX)) $(WIN32_EXE_LDFLAGS) $(LDFLAGS) $(STATIC_LIBS) $(MOZ_PROGRAM_LDFLAGS) $(SHARED_LIBS) $(OS_LIBS)
 	$(call CHECK_BINARY,$@)
 endif # WINNT && !GNU_CC
 
 ifdef ENABLE_STRIP
 	$(STRIP) $(STRIP_FLAGS) $@
 endif
 ifdef MOZ_POST_PROGRAM_COMMAND
 	$(MOZ_POST_PROGRAM_COMMAND) $@
@@ -647,17 +647,17 @@ endif
 endif
 ifndef CROSS_COMPILE
 	$(call CHECK_STDCXX,$@)
 endif
 
 $(LIBRARY): $(OBJS) $(STATIC_LIBS) $(EXTRA_DEPS) $(GLOBAL_DEPS)
 	$(REPORT_BUILD)
 	$(RM) $(REAL_LIBRARY)
-	$(AR) $(AR_FLAGS) $(OBJS) $($@_OBJS)
+	$(AR) $(AR_FLAGS) $(OBJS) $($@_$(OBJS_VAR_SUFFIX))
 
 ifeq ($(OS_ARCH),WINNT)
 # Import libraries are created by the rules creating shared libraries.
 # The rules to copy them to $(DIST)/lib depend on $(IMPORT_LIBRARY),
 # but make will happily consider the import library before it is refreshed
 # when rebuilding the corresponding shared library. Defining an empty recipe
 # for import libraries forces make to wait for the shared library recipe to
 # have run before considering other targets that depend on the import library.
@@ -675,17 +675,17 @@ endif
 # symlinks back to the originals. The symlinks are a no-op for stabs debugging,
 # so no need to conditionalize on OS version or debugging format.
 
 $(SHARED_LIBRARY): $(OBJS) $(RESFILE) $(RUST_STATIC_LIB_FOR_SHARED_LIB) $(STATIC_LIBS) $(EXTRA_DEPS) $(GLOBAL_DEPS)
 	$(REPORT_BUILD)
 ifndef INCREMENTAL_LINKER
 	$(RM) $@
 endif
-	$(MKSHLIB) $($@_OBJS) $(RESFILE) $(LDFLAGS) $(STATIC_LIBS) $(RUST_STATIC_LIB_FOR_SHARED_LIB) $(SHARED_LIBS) $(EXTRA_DSO_LDOPTS) $(MOZ_GLUE_LDFLAGS) $(OS_LIBS)
+	$(MKSHLIB) $($@_$(OBJS_VAR_SUFFIX)) $(RESFILE) $(LDFLAGS) $(STATIC_LIBS) $(RUST_STATIC_LIB_FOR_SHARED_LIB) $(SHARED_LIBS) $(EXTRA_DSO_LDOPTS) $(MOZ_GLUE_LDFLAGS) $(OS_LIBS)
 	$(call CHECK_BINARY,$@)
 
 ifeq (_WINNT,$(GNU_CC)_$(OS_ARCH))
 ifdef MSMANIFEST_TOOL
 ifdef EMBED_MANIFEST_AT
 	@if test -f $@.manifest; then \
 		if test -f '$(srcdir)/$@.manifest'; then \
 			echo 'Embedding manifest from $(srcdir)/$@.manifest and $@.manifest'; \
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -203,25 +203,31 @@ class CommonBackend(BuildBackend):
             }
             json.dump(d, fh, sort_keys=True, indent=4)
 
     def _expand_libs(self, input_bin):
         os_libs = []
         shared_libs = []
         static_libs = []
         objs = []
+        no_pgo_objs = []
 
         seen_objs = set()
         seen_libs = set()
 
         def add_objs(lib):
             for o in lib.objs:
                 if o not in seen_objs:
                     seen_objs.add(o)
                     objs.append(o)
+                    # This is slightly odd, buf for consistency with the
+                    # recursivemake backend we don't replace OBJ_SUFFIX if any
+                    # object in a library has `no_pgo` set.
+                    if lib.no_pgo_objs or lib.no_pgo:
+                        no_pgo_objs.append(o)
 
         def expand(lib, recurse_objs, system_libs):
             if isinstance(lib, StaticLibrary):
                 if lib.no_expand_lib:
                     static_libs.append(lib)
                     recurse_objs = False
                 elif recurse_objs:
                     add_objs(lib)
@@ -253,17 +259,17 @@ class CommonBackend(BuildBackend):
                     seen_libs.add(lib)
                     shared_libs.append(lib)
 
         for lib in input_bin.linked_system_libs:
             if lib not in seen_libs:
                 seen_libs.add(lib)
                 os_libs.append(lib)
 
-        return objs, shared_libs, os_libs, static_libs
+        return objs, no_pgo_objs, shared_libs, os_libs, static_libs
 
     def _make_list_file(self, objdir, objs, name):
         if not objs:
             return None
         list_style = self.environment.substs.get('EXPAND_LIBS_LIST_STYLE')
         list_file_path = mozpath.join(objdir, name)
         objs = [os.path.relpath(o, objdir) for o in objs]
         if list_style == 'linkerscript':
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -1327,43 +1327,78 @@ class RecursiveMakeBackend(CommonBackend
         def pretty_relpath(lib):
             return '$(DEPTH)/%s' % mozpath.relpath(lib.objdir, topobjdir)
 
         topobjdir = mozpath.normsep(obj.topobjdir)
         # This will create the node even if there aren't any linked libraries.
         build_target = self._build_target_for_obj(obj)
         self._compile_graph[build_target]
 
-        objs, shared_libs, os_libs, static_libs = self._expand_libs(obj)
+        objs, no_pgo_objs, shared_libs, os_libs, static_libs = self._expand_libs(obj)
 
         if obj.KIND == 'target':
             obj_target = obj.name
             if isinstance(obj, Program):
                 obj_target = self._pretty_path(obj.output_path, backend_file)
 
+            is_unit_test = isinstance(obj, BaseProgram) and obj.is_unit_test
+            profile_gen_objs = []
+
+            if (self.environment.substs.get('MOZ_PGO') and
+                self.environment.substs.get('GNU_CC')):
+                # We use a different OBJ_SUFFIX for the profile generate phase on
+                # linux. These get picked up via OBJS_VAR_SUFFIX in config.mk.
+                if not is_unit_test and not isinstance(obj, SimpleProgram):
+                    profile_gen_objs = [o if o in no_pgo_objs else '%s.%s' %
+                                        (mozpath.splitext(o)[0], 'i_o') for o in objs]
+
+            def write_obj_deps(target, objs_ref, pgo_objs_ref):
+                if pgo_objs_ref:
+                    backend_file.write('ifdef MOZ_PROFILE_GENERATE\n')
+                    backend_file.write('%s: %s\n' % (target, pgo_objs_ref))
+                    backend_file.write('else\n')
+                    backend_file.write('%s: %s\n' % (target, objs_ref))
+                    backend_file.write('endif\n')
+                else:
+                    backend_file.write('%s: %s\n' % (target, objs_ref))
+
             objs_ref = ' \\\n    '.join(os.path.relpath(o, obj.objdir)
                                         for o in objs)
+            pgo_objs_ref = ' \\\n    '.join(os.path.relpath(o, obj.objdir)
+                                            for o in profile_gen_objs)
             # Don't bother with a list file if we're only linking objects built
             # in this directory or building a real static library. This
             # accommodates clang-plugin, where we would otherwise pass an
             # incorrect list file format to the host compiler as well as when
             # creating an archive with AR, which doesn't understand list files.
             if (objs == obj.objs and not isinstance(obj, StaticLibrary) or
-                isinstance(obj, StaticLibrary) and obj.no_expand_lib):
-                backend_file.write_once('%s_OBJS := %s\n' %
-                                        (obj.name, objs_ref))
-                backend_file.write_once('%s: %s\n' % (obj_target, objs_ref))
+              isinstance(obj, StaticLibrary) and obj.no_expand_lib):
+                backend_file.write_once('%s_OBJS := %s\n' % (obj.name,
+                                                             objs_ref))
+                if profile_gen_objs:
+                    backend_file.write_once('%s_PGO_OBJS := %s\n' % (obj.name,
+                                                                     pgo_objs_ref))
+                write_obj_deps(obj_target, objs_ref, pgo_objs_ref)
             elif not isinstance(obj, StaticLibrary):
                 list_file_path = '%s.list' % obj.name.replace('.', '_')
                 list_file_ref = self._make_list_file(obj.objdir, objs,
                                                      list_file_path)
                 backend_file.write_once('%s_OBJS := %s\n' %
                                         (obj.name, list_file_ref))
                 backend_file.write_once('%s: %s\n' % (obj_target, list_file_path))
-                backend_file.write_once('%s: %s\n' % (obj_target, objs_ref))
+                if profile_gen_objs:
+                    pgo_list_file_path = '%s_pgo.list' % obj.name.replace('.', '_')
+                    pgo_list_file_ref = self._make_list_file(obj.objdir,
+                                                             profile_gen_objs,
+                                                             pgo_list_file_path)
+                    backend_file.write_once('%s_PGO_OBJS := %s\n' %
+                                            (obj.name, pgo_list_file_ref))
+                    backend_file.write_once('%s: %s\n' % (obj_target,
+                                                          pgo_list_file_path))
+                write_obj_deps(obj_target, objs_ref, pgo_objs_ref)
 
         for lib in shared_libs:
             backend_file.write_once('SHARED_LIBS += %s/%s\n' %
                                     (pretty_relpath(lib), lib.import_name))
         for lib in static_libs:
             backend_file.write_once('STATIC_LIBS += %s/%s\n' %
                                     (pretty_relpath(lib), lib.import_name))
         for lib in os_libs:
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -383,26 +383,30 @@ class LinkageMultipleRustLibrariesError(
 
 class Linkable(ContextDerived):
     """Generic context derived container object for programs and libraries"""
     __slots__ = (
         'cxx_link',
         'lib_defines',
         'linked_libraries',
         'linked_system_libs',
+        'no_pgo_sources',
+        'no_pgo',
         'sources',
     )
 
     def __init__(self, context):
         ContextDerived.__init__(self, context)
         self.cxx_link = False
         self.linked_libraries = []
         self.linked_system_libs = []
         self.lib_defines = Defines(context, {})
         self.sources = defaultdict(list)
+        self.no_pgo_sources = []
+        self.no_pgo = False
 
     def link_library(self, obj):
         assert isinstance(obj, BaseLibrary)
         if obj.KIND != self.KIND:
             raise LinkageWrongKindError('%s != %s' % (obj.KIND, self.KIND))
         # Linking multiple Rust libraries into an object would result in
         # multiple copies of the Rust standard library, as well as linking
         # errors from duplicate symbols.
@@ -432,26 +436,33 @@ class Linkable(ContextDerived):
     def source_files(self):
         all_sources = []
         # This is ordered for reproducibility and consistently w/
         # config/rules.mk
         for suffix in ('.c', '.S', '.cpp', '.m', '.mm', '.s'):
             all_sources += self.sources.get(suffix, [])
         return all_sources
 
-    @property
-    def objs(self):
+    def _get_objs(self, sources):
         obj_prefix = ''
         if self.KIND == 'host':
             obj_prefix = 'host_'
 
         return [mozpath.join(self.objdir, '%s%s.%s' % (obj_prefix,
                                                        mozpath.splitext(mozpath.basename(f))[0],
                                                        self.config.substs.get('OBJ_SUFFIX', '')))
-                for f in self.source_files()]
+                for f in sources]
+
+    @property
+    def no_pgo_objs(self):
+        return self._get_objs(self.no_pgo_sources)
+
+    @property
+    def objs(self):
+        return self._get_objs(self.source_files())
 
 
 class BaseProgram(Linkable):
     """Context derived container object for programs, which is a unicode
     string.
 
     This class handles automatically appending a binary suffix to the program
     name.
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -946,16 +946,20 @@ class TreeMetadataEmitter(LoggingMixin):
                     ctxt_sources[variable][canonical_suffix] += sorted(srcs)
                     yield obj
 
         if ctxt_sources:
             for linkable in linkables:
                 for target_var in ('SOURCES', 'UNIFIED_SOURCES'):
                     for suffix, srcs in ctxt_sources[target_var].items():
                         linkable.sources[suffix] += srcs
+                if no_pgo_sources:
+                    linkable.no_pgo_sources = no_pgo_sources
+                elif no_pgo:
+                    linkable.no_pgo = True
             for host_linkable in host_linkables:
                 for suffix, srcs in ctxt_sources['HOST_SOURCES'].items():
                     host_linkable.sources[suffix] += srcs
 
         for f, flags in all_flags.iteritems():
             if flags.flags:
                 ext = mozpath.splitext(f)[1]
                 yield PerSourceFlag(context, f, flags.flags)