Bug 921307 - Aggregate dependencies in subdirectories that are still recursed with pseudo derecurse. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Thu, 03 Oct 2013 05:51:19 +0900
changeset 163578 3cf52700118c3660e180bb5af51280fe0c33745b
parent 163577 c9d0dc574767a9ad61e7b997490d699a040a24df
child 163579 477781ec7d38292067731d10865b1dc94ec564e2
push idunknown
push userunknown
push dateunknown
reviewersgps
bugs921307
milestone27.0a1
Bug 921307 - Aggregate dependencies in subdirectories that are still recursed with pseudo derecurse. r=gps
Makefile.in
config/makefiles/target_libs.mk
config/recurse.mk
js/src/config/baseconfig.mk
js/src/config/makefiles/target_libs.mk
js/src/config/recurse.mk
python/mozbuild/mozbuild/action/link_deps.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -238,15 +238,10 @@ check::
 	$(call SUBMAKE,$@,js/src)
 endif
 
 ifdef MOZ_PSEUDO_DERECURSE
 # Interdependencies for parallel export.
 js/xpconnect/src/export: dom/bindings/export
 accessible/src/xpcom/export: xpcom/xpidl/export
 js/src/export: mfbt/export
-# Temporary interdependencies for binaries, until bug 921307 is fixed.
-# Avoid traversal of those directories when creating the aggregate dependency file
-ifneq (binaries-deps,$(MAKECMDGOALS))
-toolkit/library/binaries: js/src/binaries media/webrtc/trunk/binaries media/webrtc/signaling/binaries media/webrtc/trunk/testing/binaries media/webrtc/signalingtest/binaries media/mtransport/third_party/nrappkit/binaries media/mtransport/third_party/nICEr/binaries
 endif
 endif
-endif
--- a/config/makefiles/target_libs.mk
+++ b/config/makefiles/target_libs.mk
@@ -99,38 +99,25 @@ HOST_LIBRARY_TARGET = binaries libs
 INSTALL_TARGETS += HOST_LIBRARY
 endif
 
 endif # !NO_DIST_INSTALL
 
 ifdef MOZ_PSEUDO_DERECURSE
 BINARIES_INSTALL_TARGETS := $(foreach category,$(INSTALL_TARGETS),$(if $(filter binaries,$($(category)_TARGET)),$(category)))
 
-ifneq (,$(strip $(BINARIES_INSTALL_TARGETS)))
 # Fill a dependency file with all the binaries installed somewhere in $(DIST)
+# and with dependencies on the relevant backend files.
 BINARIES_PP := $(MDDEPDIR)/binaries.pp
 
-$(BINARIES_PP): Makefile backend.mk $(call mkdir_deps,$(MDDEPDIR))
+$(BINARIES_PP): Makefile $(if $(EXTERNALLY_MANAGED_MAKE_FILE),,backend.mk) $(call mkdir_deps,$(MDDEPDIR))
 	@echo "$(strip $(foreach category,$(BINARIES_INSTALL_TARGETS),\
 		$(foreach file,$($(category)_FILES) $($(category)_EXECUTABLES),\
 			$($(category)_DEST)/$(notdir $(file)): $(file)%\
 		)\
-	))" | tr % '\n' > $@
-endif
-
-binaries libs:: $(TARGETS) $(BINARIES_PP)
-# Aggregate all dependency files relevant to a binaries build. If there is nothing
-# done in the current directory, just create an empty stamp.
-# Externally managed make files (gyp managed) and root make files (js/src/Makefile)
-# need to be recursed to do their duty, and creating a stamp would prevent that.
-# In the future, we'll aggregate those.
-ifneq (.,$(DEPTH))
-ifndef EXTERNALLY_MANAGED_MAKE_FILE
-	@$(if $^,$(call py_action,link_deps,-o binaries --group-all --topsrcdir $(topsrcdir) --topobjdir $(DEPTH) --dist $(DIST) $(BINARIES_PP) $(wildcard $(addsuffix .pp,$(addprefix $(MDDEPDIR)/,$(notdir $(sort $(filter-out $(BINARIES_PP),$^) $(OBJ_TARGETS))))))),$(TOUCH) binaries)
-endif
-endif
+	))binaries: Makefile $(if $(EXTERNALLY_MANAGED_MAKE_FILE),,backend.mk)" | tr % '\n' > $@
 
 else
 binaries::
 	$(error The binaries target is not supported without MOZ_PSEUDO_DERECURSE)
 endif
 
 # EOF
--- a/config/recurse.mk
+++ b/config/recurse.mk
@@ -87,17 +87,17 @@ endif
 	@$(STAMP_TOUCH)
 
 GARBAGE_DIRS += subtiers
 
 # Recursion rule for all directories traversed for all subtiers in the
 # current tier.
 # root.mk defines subtier_of_* variables, that map a normalized subdir path to
 # a subtier name (e.g. subtier_of_memory_jemalloc = base)
-$(addsuffix /$(CURRENT_TIER),$(CURRENT_DIRS)): %/$(CURRENT_TIER): $(if $(WANT_STAMPS),%/Makefile %/backend.mk)
+$(addsuffix /$(CURRENT_TIER),$(CURRENT_DIRS)): %/$(CURRENT_TIER):
 ifdef BUG_915535_FIXED
 	$(call BUILDSTATUS,TIERDIR_START $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
 endif
 	+@$(MAKE) -C $* $(if $(filter $*,$(tier_$(subtier_of_$(subst /,_,$*))_staticdirs)),,$(CURRENT_TIER))
 # Ensure existing stamps are up-to-date, but don't create one if submake didn't create one.
 	$(if $(wildcard $@),@$(STAMP_TOUCH))
 ifdef BUG_915535_FIXED
 	$(call BUILDSTATUS,TIERDIR_FINISH $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
@@ -122,17 +122,17 @@ ifneq (,$(filter libs binaries,$(CURRENT
 # gyp managed ones and js/src.
 # A few things that are not traversed by a "binaries" build, but should, in an ideal
 # world, are nspr, nss, icu and ffi.
 recurse_$(CURRENT_TIER):
 	@$(MAKE) binaries-deps
 
 # Creating binaries-deps.mk directly would make us build it twice: once when beginning
 # the build because of the include, and once at the end because of the stamps.
-binaries-deps: $(wildcard $(addsuffix /binaries,$(CURRENT_DIRS)))
+binaries-deps: $(addsuffix /binaries,$(CURRENT_DIRS))
 	@$(call py_action,link_deps,-o $@.mk --group-by-depfile --topsrcdir $(topsrcdir) --topobjdir $(DEPTH) --dist $(DIST) --guard $(addprefix ",$(addsuffix ",$^)))
 	@$(TOUCH) $@
 
 ifeq (recurse_binaries,$(MAKECMDGOALS))
 $(call include_deps,binaries-deps.mk)
 endif
 
 endif
@@ -188,8 +188,32 @@ endef
 tools export:: $(SUBMAKEFILES)
 	$(LOOP_OVER_TOOL_DIRS)
 
 endif # ifdef TIERS
 
 endif # ifeq ($(EXTERNALLY_MANAGED_MAKE_FILE)_$(NO_RECURSE_MAKELEVEL),_$(MAKELEVEL))
 
 endif # ifeq (1_.,$(MOZ_PSEUDO_DERECURSE)_$(DEPTH))
+
+ifdef MOZ_PSEUDO_DERECURSE
+ifdef EXTERNALLY_MANAGED_MAKE_FILE
+# gyp-managed directories
+recurse_targets := $(addsuffix /binaries,$(DIRS) $(PARALLEL_DIRS))
+else
+ifeq (.,$(DEPTH))
+# top-level directories
+recurse_targets := $(addsuffix /binaries,$(binaries_dirs))
+ifdef recurse_targets
+# only js/src has binaries_dirs, and we want to adjust paths for it.
+want_abspaths = 1
+endif
+endif
+endif
+
+binaries libs:: $(TARGETS) $(BINARIES_PP)
+# Aggregate all dependency files relevant to a binaries build except in
+# the mozilla top-level directory.
+ifneq (_.,$(recurse_targets)_$(DEPTH))
+	@$(if $(or $(recurse_targets),$^),$(call py_action,link_deps,-o binaries --group-all $(if $(want_abspaths),--abspaths )--topsrcdir $(topsrcdir) --topobjdir $(DEPTH) --dist $(DIST) $(BINARIES_PP) $(wildcard $(addsuffix .pp,$(addprefix $(MDDEPDIR)/,$(notdir $(sort $(filter-out $(BINARIES_PP),$^) $(OBJ_TARGETS)))))) $(recurse_targets)))
+endif
+
+endif # ifdef MOZ_PSEUDO_DERECURSE
--- a/js/src/config/baseconfig.mk
+++ b/js/src/config/baseconfig.mk
@@ -1,16 +1,20 @@
 installdir = $(libdir)/$(MOZ_APP_NAME)-$(MOZ_APP_VERSION)
 sdkdir = $(libdir)/$(MOZ_APP_NAME)-devel-$(MOZ_APP_VERSION)
 
 ifneq (,$(filter /%,$(TOP_DIST)))
 DIST = $(TOP_DIST)
 else
+ifeq (.,$(DEPTH))
+DIST = $(TOP_DIST)
+else
 DIST = $(DEPTH)/$(TOP_DIST)
 endif
+endif
 
 # We do magic with OBJ_SUFFIX in config.mk, the following ensures we don't
 # manually use it before config.mk inclusion
 _OBJ_SUFFIX := $(OBJ_SUFFIX)
 OBJ_SUFFIX = $(error config/config.mk needs to be included before using OBJ_SUFFIX)
 
 # We only want to do the pymake sanity on Windows, other os's can cope
 ifeq ($(HOST_OS_ARCH),WINNT)
--- a/js/src/config/makefiles/target_libs.mk
+++ b/js/src/config/makefiles/target_libs.mk
@@ -99,38 +99,25 @@ HOST_LIBRARY_TARGET = binaries libs
 INSTALL_TARGETS += HOST_LIBRARY
 endif
 
 endif # !NO_DIST_INSTALL
 
 ifdef MOZ_PSEUDO_DERECURSE
 BINARIES_INSTALL_TARGETS := $(foreach category,$(INSTALL_TARGETS),$(if $(filter binaries,$($(category)_TARGET)),$(category)))
 
-ifneq (,$(strip $(BINARIES_INSTALL_TARGETS)))
 # Fill a dependency file with all the binaries installed somewhere in $(DIST)
+# and with dependencies on the relevant backend files.
 BINARIES_PP := $(MDDEPDIR)/binaries.pp
 
-$(BINARIES_PP): Makefile backend.mk $(call mkdir_deps,$(MDDEPDIR))
+$(BINARIES_PP): Makefile $(if $(EXTERNALLY_MANAGED_MAKE_FILE),,backend.mk) $(call mkdir_deps,$(MDDEPDIR))
 	@echo "$(strip $(foreach category,$(BINARIES_INSTALL_TARGETS),\
 		$(foreach file,$($(category)_FILES) $($(category)_EXECUTABLES),\
 			$($(category)_DEST)/$(notdir $(file)): $(file)%\
 		)\
-	))" | tr % '\n' > $@
-endif
-
-binaries libs:: $(TARGETS) $(BINARIES_PP)
-# Aggregate all dependency files relevant to a binaries build. If there is nothing
-# done in the current directory, just create an empty stamp.
-# Externally managed make files (gyp managed) and root make files (js/src/Makefile)
-# need to be recursed to do their duty, and creating a stamp would prevent that.
-# In the future, we'll aggregate those.
-ifneq (.,$(DEPTH))
-ifndef EXTERNALLY_MANAGED_MAKE_FILE
-	@$(if $^,$(call py_action,link_deps,-o binaries --group-all --topsrcdir $(topsrcdir) --topobjdir $(DEPTH) --dist $(DIST) $(BINARIES_PP) $(wildcard $(addsuffix .pp,$(addprefix $(MDDEPDIR)/,$(notdir $(sort $(filter-out $(BINARIES_PP),$^) $(OBJ_TARGETS))))))),$(TOUCH) binaries)
-endif
-endif
+	))binaries: Makefile $(if $(EXTERNALLY_MANAGED_MAKE_FILE),,backend.mk)" | tr % '\n' > $@
 
 else
 binaries::
 	$(error The binaries target is not supported without MOZ_PSEUDO_DERECURSE)
 endif
 
 # EOF
--- a/js/src/config/recurse.mk
+++ b/js/src/config/recurse.mk
@@ -87,17 +87,17 @@ endif
 	@$(STAMP_TOUCH)
 
 GARBAGE_DIRS += subtiers
 
 # Recursion rule for all directories traversed for all subtiers in the
 # current tier.
 # root.mk defines subtier_of_* variables, that map a normalized subdir path to
 # a subtier name (e.g. subtier_of_memory_jemalloc = base)
-$(addsuffix /$(CURRENT_TIER),$(CURRENT_DIRS)): %/$(CURRENT_TIER): $(if $(WANT_STAMPS),%/Makefile %/backend.mk)
+$(addsuffix /$(CURRENT_TIER),$(CURRENT_DIRS)): %/$(CURRENT_TIER):
 ifdef BUG_915535_FIXED
 	$(call BUILDSTATUS,TIERDIR_START $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
 endif
 	+@$(MAKE) -C $* $(if $(filter $*,$(tier_$(subtier_of_$(subst /,_,$*))_staticdirs)),,$(CURRENT_TIER))
 # Ensure existing stamps are up-to-date, but don't create one if submake didn't create one.
 	$(if $(wildcard $@),@$(STAMP_TOUCH))
 ifdef BUG_915535_FIXED
 	$(call BUILDSTATUS,TIERDIR_FINISH $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
@@ -122,17 +122,17 @@ ifneq (,$(filter libs binaries,$(CURRENT
 # gyp managed ones and js/src.
 # A few things that are not traversed by a "binaries" build, but should, in an ideal
 # world, are nspr, nss, icu and ffi.
 recurse_$(CURRENT_TIER):
 	@$(MAKE) binaries-deps
 
 # Creating binaries-deps.mk directly would make us build it twice: once when beginning
 # the build because of the include, and once at the end because of the stamps.
-binaries-deps: $(wildcard $(addsuffix /binaries,$(CURRENT_DIRS)))
+binaries-deps: $(addsuffix /binaries,$(CURRENT_DIRS))
 	@$(call py_action,link_deps,-o $@.mk --group-by-depfile --topsrcdir $(topsrcdir) --topobjdir $(DEPTH) --dist $(DIST) --guard $(addprefix ",$(addsuffix ",$^)))
 	@$(TOUCH) $@
 
 ifeq (recurse_binaries,$(MAKECMDGOALS))
 $(call include_deps,binaries-deps.mk)
 endif
 
 endif
@@ -188,8 +188,32 @@ endef
 tools export:: $(SUBMAKEFILES)
 	$(LOOP_OVER_TOOL_DIRS)
 
 endif # ifdef TIERS
 
 endif # ifeq ($(EXTERNALLY_MANAGED_MAKE_FILE)_$(NO_RECURSE_MAKELEVEL),_$(MAKELEVEL))
 
 endif # ifeq (1_.,$(MOZ_PSEUDO_DERECURSE)_$(DEPTH))
+
+ifdef MOZ_PSEUDO_DERECURSE
+ifdef EXTERNALLY_MANAGED_MAKE_FILE
+# gyp-managed directories
+recurse_targets := $(addsuffix /binaries,$(DIRS) $(PARALLEL_DIRS))
+else
+ifeq (.,$(DEPTH))
+# top-level directories
+recurse_targets := $(addsuffix /binaries,$(binaries_dirs))
+ifdef recurse_targets
+# only js/src has binaries_dirs, and we want to adjust paths for it.
+want_abspaths = 1
+endif
+endif
+endif
+
+binaries libs:: $(TARGETS) $(BINARIES_PP)
+# Aggregate all dependency files relevant to a binaries build except in
+# the mozilla top-level directory.
+ifneq (_.,$(recurse_targets)_$(DEPTH))
+	@$(if $(or $(recurse_targets),$^),$(call py_action,link_deps,-o binaries --group-all $(if $(want_abspaths),--abspaths )--topsrcdir $(topsrcdir) --topobjdir $(DEPTH) --dist $(DIST) $(BINARIES_PP) $(wildcard $(addsuffix .pp,$(addprefix $(MDDEPDIR)/,$(notdir $(sort $(filter-out $(BINARIES_PP),$^) $(OBJ_TARGETS)))))) $(recurse_targets)))
+endif
+
+endif # ifdef MOZ_PSEUDO_DERECURSE
--- a/python/mozbuild/mozbuild/action/link_deps.py
+++ b/python/mozbuild/mozbuild/action/link_deps.py
@@ -79,32 +79,42 @@ import mozpack.path as mozpath
 def enum(**enums):
     return type('Enum', (), enums)
 
 
 Grouping = enum(NO=0, BY_DEPFILE=1, ALL_TARGETS=2)
 
 
 class DependencyLinker(Makefile):
-    def __init__(self, topsrcdir, topobjdir, dist, group=Grouping.NO):
+    def __init__(self, topsrcdir, topobjdir, dist, group=Grouping.NO,
+                 abspaths=False):
         topsrcdir = mozpath.normsep(os.path.normcase(os.path.abspath(topsrcdir)))
         topobjdir = mozpath.normsep(os.path.normcase(os.path.abspath(topobjdir)))
         dist = mozpath.normsep(os.path.normcase(os.path.abspath(dist)))
+        if abspaths:
+            topsrcdir_value = topsrcdir
+            topobjdir_value = topobjdir
+            dist_value = dist
+        else:
+            topsrcdir_value = '$(topsrcdir)'
+            topobjdir_value = '$(DEPTH)'
+            dist_value = '$(DIST)'
+
         self._normpaths = {
-            topsrcdir: '$(topsrcdir)',
-            topobjdir: '$(DEPTH)',
-            dist: '$(DIST)',
-            '$(topsrcdir)': '$(topsrcdir)',
-            '$(DEPTH)': '$(DEPTH)',
-            '$(DIST)': '$(DIST)',
-            '$(depth)': '$(DEPTH)', # normcase may lowercase variable refs when
-            '$(dist)': '$(DIST)',   # they are in the original dependency file
-            mozpath.relpath(topsrcdir, os.curdir): '$(topsrcdir)',
-            mozpath.relpath(topobjdir, os.curdir): '$(DEPTH)',
-            mozpath.relpath(dist, os.curdir): '$(DIST)',
+            topsrcdir: topsrcdir_value,
+            topobjdir: topobjdir_value,
+            dist: dist_value,
+            '$(topsrcdir)': topsrcdir_value,
+            '$(DEPTH)': topobjdir_value,
+            '$(DIST)': dist_value,
+            '$(depth)': topobjdir_value, # normcase may lowercase variable refs when
+            '$(dist)': dist_value,       # they are in the original dependency file
+            mozpath.relpath(topsrcdir, os.curdir): topsrcdir_value,
+            mozpath.relpath(topobjdir, os.curdir): topobjdir_value,
+            mozpath.relpath(dist, os.curdir): dist_value,
         }
 
         Makefile.__init__(self)
         self._group = group
         self._targets = OrderedDict()
 
     def add_dependencies(self, fh):
         depfile = self.normpath(os.path.abspath(fh.name))
@@ -179,28 +189,31 @@ def main(args):
     parser.add_argument('dependency_files', nargs='*')
     parser.add_argument('--guard', action="store_true",
         help='Add removal guards in the linked result.')
     group = parser.add_mutually_exclusive_group()
     group.add_argument('--group-by-depfile', action='store_true',
         help='Group dependencies by depfile.')
     group.add_argument('--group-all', action='store_true',
         help='Group all dependencies under one target.')
+    parser.add_argument('--abspaths', action='store_true',
+        help='Use absolute paths instead of using make variable references.')
     opts = parser.parse_args(args)
 
     if opts.group_by_depfile:
         group = Grouping.BY_DEPFILE
     elif opts.group_all:
         group = Grouping.ALL_TARGETS
     else:
         group = Grouping.NO
     linker = DependencyLinker(topsrcdir=opts.topsrcdir,
                               topobjdir=opts.topobjdir,
                               dist=opts.dist,
-                              group=group)
+                              group=group,
+                              abspaths=opts.abspaths)
     for f in opts.dependency_files:
         linker.add_dependencies(open(f))
 
     if opts.output:
         out = open(opts.output, 'w')
     else:
         out = sys.stdout
     linker.dump(out, removal_guard=opts.guard)