Bug 921307 - Aggregate dependencies in subdirectories that are still recursed with pseudo derecurse. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Thu, 03 Oct 2013 05:51:19 +0900
changeset 149702 3cf52700118c3660e180bb5af51280fe0c33745b
parent 149701 c9d0dc574767a9ad61e7b997490d699a040a24df
child 149703 477781ec7d38292067731d10865b1dc94ec564e2
push id34654
push usermh@glandium.org
push dateWed, 02 Oct 2013 20:52:46 +0000
treeherdermozilla-inbound@3cf52700118c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs921307
milestone27.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 921307 - Aggregate dependencies in subdirectories that are still recursed with pseudo derecurse. r=gps
Makefile.in
config/makefiles/target_libs.mk
config/recurse.mk
js/src/config/baseconfig.mk
js/src/config/makefiles/target_libs.mk
js/src/config/recurse.mk
python/mozbuild/mozbuild/action/link_deps.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -238,15 +238,10 @@ check::
 	$(call SUBMAKE,$@,js/src)
 endif
 
 ifdef MOZ_PSEUDO_DERECURSE
 # Interdependencies for parallel export.
 js/xpconnect/src/export: dom/bindings/export
 accessible/src/xpcom/export: xpcom/xpidl/export
 js/src/export: mfbt/export
-# Temporary interdependencies for binaries, until bug 921307 is fixed.
-# Avoid traversal of those directories when creating the aggregate dependency file
-ifneq (binaries-deps,$(MAKECMDGOALS))
-toolkit/library/binaries: js/src/binaries media/webrtc/trunk/binaries media/webrtc/signaling/binaries media/webrtc/trunk/testing/binaries media/webrtc/signalingtest/binaries media/mtransport/third_party/nrappkit/binaries media/mtransport/third_party/nICEr/binaries
 endif
 endif
-endif
--- a/config/makefiles/target_libs.mk
+++ b/config/makefiles/target_libs.mk
@@ -99,38 +99,25 @@ HOST_LIBRARY_TARGET = binaries libs
 INSTALL_TARGETS += HOST_LIBRARY
 endif
 
 endif # !NO_DIST_INSTALL
 
 ifdef MOZ_PSEUDO_DERECURSE
 BINARIES_INSTALL_TARGETS := $(foreach category,$(INSTALL_TARGETS),$(if $(filter binaries,$($(category)_TARGET)),$(category)))
 
-ifneq (,$(strip $(BINARIES_INSTALL_TARGETS)))
 # Fill a dependency file with all the binaries installed somewhere in $(DIST)
+# and with dependencies on the relevant backend files.
 BINARIES_PP := $(MDDEPDIR)/binaries.pp
 
-$(BINARIES_PP): Makefile backend.mk $(call mkdir_deps,$(MDDEPDIR))
+$(BINARIES_PP): Makefile $(if $(EXTERNALLY_MANAGED_MAKE_FILE),,backend.mk) $(call mkdir_deps,$(MDDEPDIR))
 	@echo "$(strip $(foreach category,$(BINARIES_INSTALL_TARGETS),\
 		$(foreach file,$($(category)_FILES) $($(category)_EXECUTABLES),\
 			$($(category)_DEST)/$(notdir $(file)): $(file)%\
 		)\
-	))" | tr % '\n' > $@
-endif
-
-binaries libs:: $(TARGETS) $(BINARIES_PP)
-# Aggregate all dependency files relevant to a binaries build. If there is nothing
-# done in the current directory, just create an empty stamp.
-# Externally managed make files (gyp managed) and root make files (js/src/Makefile)
-# need to be recursed to do their duty, and creating a stamp would prevent that.
-# In the future, we'll aggregate those.
-ifneq (.,$(DEPTH))
-ifndef EXTERNALLY_MANAGED_MAKE_FILE
-	@$(if $^,$(call py_action,link_deps,-o binaries --group-all --topsrcdir $(topsrcdir) --topobjdir $(DEPTH) --dist $(DIST) $(BINARIES_PP) $(wildcard $(addsuffix .pp,$(addprefix $(MDDEPDIR)/,$(notdir $(sort $(filter-out $(BINARIES_PP),$^) $(OBJ_TARGETS))))))),$(TOUCH) binaries)
-endif
-endif
+	))binaries: Makefile $(if $(EXTERNALLY_MANAGED_MAKE_FILE),,backend.mk)" | tr % '\n' > $@
 
 else
 binaries::
 	$(error The binaries target is not supported without MOZ_PSEUDO_DERECURSE)
 endif
 
 # EOF
--- a/config/recurse.mk
+++ b/config/recurse.mk
@@ -87,17 +87,17 @@ endif
 	@$(STAMP_TOUCH)
 
 GARBAGE_DIRS += subtiers
 
 # Recursion rule for all directories traversed for all subtiers in the
 # current tier.
 # root.mk defines subtier_of_* variables, that map a normalized subdir path to
 # a subtier name (e.g. subtier_of_memory_jemalloc = base)
-$(addsuffix /$(CURRENT_TIER),$(CURRENT_DIRS)): %/$(CURRENT_TIER): $(if $(WANT_STAMPS),%/Makefile %/backend.mk)
+$(addsuffix /$(CURRENT_TIER),$(CURRENT_DIRS)): %/$(CURRENT_TIER):
 ifdef BUG_915535_FIXED
 	$(call BUILDSTATUS,TIERDIR_START $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
 endif
 	+@$(MAKE) -C $* $(if $(filter $*,$(tier_$(subtier_of_$(subst /,_,$*))_staticdirs)),,$(CURRENT_TIER))
 # Ensure existing stamps are up-to-date, but don't create one if submake didn't create one.
 	$(if $(wildcard $@),@$(STAMP_TOUCH))
 ifdef BUG_915535_FIXED
 	$(call BUILDSTATUS,TIERDIR_FINISH $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
@@ -122,17 +122,17 @@ ifneq (,$(filter libs binaries,$(CURRENT
 # gyp managed ones and js/src.
 # A few things that are not traversed by a "binaries" build, but should, in an ideal
 # world, are nspr, nss, icu and ffi.
 recurse_$(CURRENT_TIER):
 	@$(MAKE) binaries-deps
 
 # Creating binaries-deps.mk directly would make us build it twice: once when beginning
 # the build because of the include, and once at the end because of the stamps.
-binaries-deps: $(wildcard $(addsuffix /binaries,$(CURRENT_DIRS)))
+binaries-deps: $(addsuffix /binaries,$(CURRENT_DIRS))
 	@$(call py_action,link_deps,-o $@.mk --group-by-depfile --topsrcdir $(topsrcdir) --topobjdir $(DEPTH) --dist $(DIST) --guard $(addprefix ",$(addsuffix ",$^)))
 	@$(TOUCH) $@
 
 ifeq (recurse_binaries,$(MAKECMDGOALS))
 $(call include_deps,binaries-deps.mk)
 endif
 
 endif
@@ -188,8 +188,32 @@ endef
 tools export:: $(SUBMAKEFILES)
 	$(LOOP_OVER_TOOL_DIRS)
 
 endif # ifdef TIERS
 
 endif # ifeq ($(EXTERNALLY_MANAGED_MAKE_FILE)_$(NO_RECURSE_MAKELEVEL),_$(MAKELEVEL))
 
 endif # ifeq (1_.,$(MOZ_PSEUDO_DERECURSE)_$(DEPTH))
+
+ifdef MOZ_PSEUDO_DERECURSE
+ifdef EXTERNALLY_MANAGED_MAKE_FILE
+# gyp-managed directories
+recurse_targets := $(addsuffix /binaries,$(DIRS) $(PARALLEL_DIRS))
+else
+ifeq (.,$(DEPTH))
+# top-level directories
+recurse_targets := $(addsuffix /binaries,$(binaries_dirs))
+ifdef recurse_targets
+# only js/src has binaries_dirs, and we want to adjust paths for it.
+want_abspaths = 1
+endif
+endif
+endif
+
+binaries libs:: $(TARGETS) $(BINARIES_PP)
+# Aggregate all dependency files relevant to a binaries build except in
+# the mozilla top-level directory.
+ifneq (_.,$(recurse_targets)_$(DEPTH))
+	@$(if $(or $(recurse_targets),$^),$(call py_action,link_deps,-o binaries --group-all $(if $(want_abspaths),--abspaths )--topsrcdir $(topsrcdir) --topobjdir $(DEPTH) --dist $(DIST) $(BINARIES_PP) $(wildcard $(addsuffix .pp,$(addprefix $(MDDEPDIR)/,$(notdir $(sort $(filter-out $(BINARIES_PP),$^) $(OBJ_TARGETS)))))) $(recurse_targets)))
+endif
+
+endif # ifdef MOZ_PSEUDO_DERECURSE
--- a/js/src/config/baseconfig.mk
+++ b/js/src/config/baseconfig.mk
@@ -1,16 +1,20 @@
 installdir = $(libdir)/$(MOZ_APP_NAME)-$(MOZ_APP_VERSION)
 sdkdir = $(libdir)/$(MOZ_APP_NAME)-devel-$(MOZ_APP_VERSION)
 
 ifneq (,$(filter /%,$(TOP_DIST)))
 DIST = $(TOP_DIST)
 else
+ifeq (.,$(DEPTH))
+DIST = $(TOP_DIST)
+else
 DIST = $(DEPTH)/$(TOP_DIST)
 endif
+endif
 
 # We do magic with OBJ_SUFFIX in config.mk, the following ensures we don't
 # manually use it before config.mk inclusion
 _OBJ_SUFFIX := $(OBJ_SUFFIX)
 OBJ_SUFFIX = $(error config/config.mk needs to be included before using OBJ_SUFFIX)
 
 # We only want to do the pymake sanity on Windows, other os's can cope
 ifeq ($(HOST_OS_ARCH),WINNT)
--- a/js/src/config/makefiles/target_libs.mk
+++ b/js/src/config/makefiles/target_libs.mk
@@ -99,38 +99,25 @@ HOST_LIBRARY_TARGET = binaries libs
 INSTALL_TARGETS += HOST_LIBRARY
 endif
 
 endif # !NO_DIST_INSTALL
 
 ifdef MOZ_PSEUDO_DERECURSE
 BINARIES_INSTALL_TARGETS := $(foreach category,$(INSTALL_TARGETS),$(if $(filter binaries,$($(category)_TARGET)),$(category)))
 
-ifneq (,$(strip $(BINARIES_INSTALL_TARGETS)))
 # Fill a dependency file with all the binaries installed somewhere in $(DIST)
+# and with dependencies on the relevant backend files.
 BINARIES_PP := $(MDDEPDIR)/binaries.pp
 
-$(BINARIES_PP): Makefile backend.mk $(call mkdir_deps,$(MDDEPDIR))
+$(BINARIES_PP): Makefile $(if $(EXTERNALLY_MANAGED_MAKE_FILE),,backend.mk) $(call mkdir_deps,$(MDDEPDIR))
 	@echo "$(strip $(foreach category,$(BINARIES_INSTALL_TARGETS),\
 		$(foreach file,$($(category)_FILES) $($(category)_EXECUTABLES),\
 			$($(category)_DEST)/$(notdir $(file)): $(file)%\
 		)\
-	))" | tr % '\n' > $@
-endif
-
-binaries libs:: $(TARGETS) $(BINARIES_PP)
-# Aggregate all dependency files relevant to a binaries build. If there is nothing
-# done in the current directory, just create an empty stamp.
-# Externally managed make files (gyp managed) and root make files (js/src/Makefile)
-# need to be recursed to do their duty, and creating a stamp would prevent that.
-# In the future, we'll aggregate those.
-ifneq (.,$(DEPTH))
-ifndef EXTERNALLY_MANAGED_MAKE_FILE
-	@$(if $^,$(call py_action,link_deps,-o binaries --group-all --topsrcdir $(topsrcdir) --topobjdir $(DEPTH) --dist $(DIST) $(BINARIES_PP) $(wildcard $(addsuffix .pp,$(addprefix $(MDDEPDIR)/,$(notdir $(sort $(filter-out $(BINARIES_PP),$^) $(OBJ_TARGETS))))))),$(TOUCH) binaries)
-endif
-endif
+	))binaries: Makefile $(if $(EXTERNALLY_MANAGED_MAKE_FILE),,backend.mk)" | tr % '\n' > $@
 
 else
 binaries::
 	$(error The binaries target is not supported without MOZ_PSEUDO_DERECURSE)
 endif
 
 # EOF
--- a/js/src/config/recurse.mk
+++ b/js/src/config/recurse.mk
@@ -87,17 +87,17 @@ endif
 	@$(STAMP_TOUCH)
 
 GARBAGE_DIRS += subtiers
 
 # Recursion rule for all directories traversed for all subtiers in the
 # current tier.
 # root.mk defines subtier_of_* variables, that map a normalized subdir path to
 # a subtier name (e.g. subtier_of_memory_jemalloc = base)
-$(addsuffix /$(CURRENT_TIER),$(CURRENT_DIRS)): %/$(CURRENT_TIER): $(if $(WANT_STAMPS),%/Makefile %/backend.mk)
+$(addsuffix /$(CURRENT_TIER),$(CURRENT_DIRS)): %/$(CURRENT_TIER):
 ifdef BUG_915535_FIXED
 	$(call BUILDSTATUS,TIERDIR_START $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
 endif
 	+@$(MAKE) -C $* $(if $(filter $*,$(tier_$(subtier_of_$(subst /,_,$*))_staticdirs)),,$(CURRENT_TIER))
 # Ensure existing stamps are up-to-date, but don't create one if submake didn't create one.
 	$(if $(wildcard $@),@$(STAMP_TOUCH))
 ifdef BUG_915535_FIXED
 	$(call BUILDSTATUS,TIERDIR_FINISH $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
@@ -122,17 +122,17 @@ ifneq (,$(filter libs binaries,$(CURRENT
 # gyp managed ones and js/src.
 # A few things that are not traversed by a "binaries" build, but should, in an ideal
 # world, are nspr, nss, icu and ffi.
 recurse_$(CURRENT_TIER):
 	@$(MAKE) binaries-deps
 
 # Creating binaries-deps.mk directly would make us build it twice: once when beginning
 # the build because of the include, and once at the end because of the stamps.
-binaries-deps: $(wildcard $(addsuffix /binaries,$(CURRENT_DIRS)))
+binaries-deps: $(addsuffix /binaries,$(CURRENT_DIRS))
 	@$(call py_action,link_deps,-o $@.mk --group-by-depfile --topsrcdir $(topsrcdir) --topobjdir $(DEPTH) --dist $(DIST) --guard $(addprefix ",$(addsuffix ",$^)))
 	@$(TOUCH) $@
 
 ifeq (recurse_binaries,$(MAKECMDGOALS))
 $(call include_deps,binaries-deps.mk)
 endif
 
 endif
@@ -188,8 +188,32 @@ endef
 tools export:: $(SUBMAKEFILES)
 	$(LOOP_OVER_TOOL_DIRS)
 
 endif # ifdef TIERS
 
 endif # ifeq ($(EXTERNALLY_MANAGED_MAKE_FILE)_$(NO_RECURSE_MAKELEVEL),_$(MAKELEVEL))
 
 endif # ifeq (1_.,$(MOZ_PSEUDO_DERECURSE)_$(DEPTH))
+
+ifdef MOZ_PSEUDO_DERECURSE
+ifdef EXTERNALLY_MANAGED_MAKE_FILE
+# gyp-managed directories
+recurse_targets := $(addsuffix /binaries,$(DIRS) $(PARALLEL_DIRS))
+else
+ifeq (.,$(DEPTH))
+# top-level directories
+recurse_targets := $(addsuffix /binaries,$(binaries_dirs))
+ifdef recurse_targets
+# only js/src has binaries_dirs, and we want to adjust paths for it.
+want_abspaths = 1
+endif
+endif
+endif
+
+binaries libs:: $(TARGETS) $(BINARIES_PP)
+# Aggregate all dependency files relevant to a binaries build except in
+# the mozilla top-level directory.
+ifneq (_.,$(recurse_targets)_$(DEPTH))
+	@$(if $(or $(recurse_targets),$^),$(call py_action,link_deps,-o binaries --group-all $(if $(want_abspaths),--abspaths )--topsrcdir $(topsrcdir) --topobjdir $(DEPTH) --dist $(DIST) $(BINARIES_PP) $(wildcard $(addsuffix .pp,$(addprefix $(MDDEPDIR)/,$(notdir $(sort $(filter-out $(BINARIES_PP),$^) $(OBJ_TARGETS)))))) $(recurse_targets)))
+endif
+
+endif # ifdef MOZ_PSEUDO_DERECURSE
--- a/python/mozbuild/mozbuild/action/link_deps.py
+++ b/python/mozbuild/mozbuild/action/link_deps.py
@@ -79,32 +79,42 @@ import mozpack.path as mozpath
 def enum(**enums):
     return type('Enum', (), enums)
 
 
 Grouping = enum(NO=0, BY_DEPFILE=1, ALL_TARGETS=2)
 
 
 class DependencyLinker(Makefile):
-    def __init__(self, topsrcdir, topobjdir, dist, group=Grouping.NO):
+    def __init__(self, topsrcdir, topobjdir, dist, group=Grouping.NO,
+                 abspaths=False):
         topsrcdir = mozpath.normsep(os.path.normcase(os.path.abspath(topsrcdir)))
         topobjdir = mozpath.normsep(os.path.normcase(os.path.abspath(topobjdir)))
         dist = mozpath.normsep(os.path.normcase(os.path.abspath(dist)))
+        if abspaths:
+            topsrcdir_value = topsrcdir
+            topobjdir_value = topobjdir
+            dist_value = dist
+        else:
+            topsrcdir_value = '$(topsrcdir)'
+            topobjdir_value = '$(DEPTH)'
+            dist_value = '$(DIST)'
+
         self._normpaths = {
-            topsrcdir: '$(topsrcdir)',
-            topobjdir: '$(DEPTH)',
-            dist: '$(DIST)',
-            '$(topsrcdir)': '$(topsrcdir)',
-            '$(DEPTH)': '$(DEPTH)',
-            '$(DIST)': '$(DIST)',
-            '$(depth)': '$(DEPTH)', # normcase may lowercase variable refs when
-            '$(dist)': '$(DIST)',   # they are in the original dependency file
-            mozpath.relpath(topsrcdir, os.curdir): '$(topsrcdir)',
-            mozpath.relpath(topobjdir, os.curdir): '$(DEPTH)',
-            mozpath.relpath(dist, os.curdir): '$(DIST)',
+            topsrcdir: topsrcdir_value,
+            topobjdir: topobjdir_value,
+            dist: dist_value,
+            '$(topsrcdir)': topsrcdir_value,
+            '$(DEPTH)': topobjdir_value,
+            '$(DIST)': dist_value,
+            '$(depth)': topobjdir_value, # normcase may lowercase variable refs when
+            '$(dist)': dist_value,       # they are in the original dependency file
+            mozpath.relpath(topsrcdir, os.curdir): topsrcdir_value,
+            mozpath.relpath(topobjdir, os.curdir): topobjdir_value,
+            mozpath.relpath(dist, os.curdir): dist_value,
         }
 
         Makefile.__init__(self)
         self._group = group
         self._targets = OrderedDict()
 
     def add_dependencies(self, fh):
         depfile = self.normpath(os.path.abspath(fh.name))
@@ -179,28 +189,31 @@ def main(args):
     parser.add_argument('dependency_files', nargs='*')
     parser.add_argument('--guard', action="store_true",
         help='Add removal guards in the linked result.')
     group = parser.add_mutually_exclusive_group()
     group.add_argument('--group-by-depfile', action='store_true',
         help='Group dependencies by depfile.')
     group.add_argument('--group-all', action='store_true',
         help='Group all dependencies under one target.')
+    parser.add_argument('--abspaths', action='store_true',
+        help='Use absolute paths instead of using make variable references.')
     opts = parser.parse_args(args)
 
     if opts.group_by_depfile:
         group = Grouping.BY_DEPFILE
     elif opts.group_all:
         group = Grouping.ALL_TARGETS
     else:
         group = Grouping.NO
     linker = DependencyLinker(topsrcdir=opts.topsrcdir,
                               topobjdir=opts.topobjdir,
                               dist=opts.dist,
-                              group=group)
+                              group=group,
+                              abspaths=opts.abspaths)
     for f in opts.dependency_files:
         linker.add_dependencies(open(f))
 
     if opts.output:
         out = open(opts.output, 'w')
     else:
         out = sys.stdout
     linker.dump(out, removal_guard=opts.guard)