Bug 915648 - Parallelize make export. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 20 Sep 2013 13:30:17 +0900
changeset 148113 9cc7634555c3e7a5d206cdd90d08661f6504853a
parent 148112 0d19104a7002076c73555a1b860d2c1ba4bcca73
child 148114 9eba1b4d798f74a5a0046c54cc12cbbfbc69b1bc
push id1
push userroot
push dateMon, 20 Oct 2014 17:29:22 +0000
reviewersgps
bugs915648
milestone27.0a1
Bug 915648 - Parallelize make export. r=gps
Makefile.in
config/recurse.mk
js/src/config/recurse.mk
moz.build
python/mozbuild/mozbuild/backend/recursivemake.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -212,8 +212,15 @@ endif
 js/src/Makefile: subsrcdir := js/src
 
 ifdef ENABLE_TESTS
 # Incorporate static tier directories into tests. This should be incorporated
 # into moz.build files someday.
 check::
 	$(call SUBMAKE,$@,js/src)
 endif
+
+ifdef MOZ_PSEUDO_DERECURSE
+# Interdependencies for parallel export.
+js/xpconnect/src/export: config/makefiles/precompile/export
+accessible/src/xpcom/export: config/makefiles/precompile/export
+js/src/export: mfbt/export
+endif
--- a/config/recurse.mk
+++ b/config/recurse.mk
@@ -17,17 +17,18 @@ endif
 #     Entering qux
 #
 # Pseudo derecurse transforms the above into:
 #   make -C foo
 #   make -C foo/bar
 #   make -C foo/baz
 #   make -C qux
 
-ifeq (1_.,$(MOZ_PSEUDO_DERECURSE)_$(DEPTH))
+# MOZ_PSEUDO_DERECURSE can have values other than 1.
+ifeq (1_.,$(if $(MOZ_PSEUDO_DERECURSE),1)_$(DEPTH))
 
 include root.mk
 
 # Disable build status for mach in top directories without TIERS.
 # In practice this disables it when recursing under js/src, which confuses mach.
 ifndef TIERS
 BUILDSTATUS =
 endif
@@ -83,16 +84,22 @@ CURRENT_DIRS := $(or $($(CURRENT_TIER)_d
 ifdef BUG_915535_FIXED
 	$(call BUILDSTATUS,TIERDIR_START $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
 endif
 	+@$(MAKE) -C $* $(if $(filter $*,$(tier_$(subtier_of_$(subst /,_,$*))_staticdirs)),,$(CURRENT_TIER))
 ifdef BUG_915535_FIXED
 	$(call BUILDSTATUS,TIERDIR_FINISH $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
 endif
 
+# The export tier requires nsinstall, which is built from config. So every
+# subdirectory traversal needs to happen after traversing config.
+ifeq ($(CURRENT_TIER),export)
+$(addsuffix /$(CURRENT_TIER),$(filter-out config,$(CURRENT_DIRS))): config/$(CURRENT_TIER)
+endif
+
 else
 
 # Don't recurse if MAKELEVEL is NO_RECURSE_MAKELEVEL as defined above, but
 # still recurse for externally managed make files (gyp-generated ones).
 ifeq ($(EXTERNALLY_MANAGED_MAKE_FILE)_$(NO_RECURSE_MAKELEVEL),_$(MAKELEVEL))
 
 compile libs export tools::
 
--- a/js/src/config/recurse.mk
+++ b/js/src/config/recurse.mk
@@ -83,16 +83,22 @@ CURRENT_DIRS := $(or $($(CURRENT_TIER)_d
 ifdef BUG_915535_FIXED
 	$(call BUILDSTATUS,TIERDIR_START $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
 endif
 	+@$(MAKE) -C $* $(if $(filter $*,$(tier_$(subtier_of_$(subst /,_,$*))_staticdirs)),,$(CURRENT_TIER))
 ifdef BUG_915535_FIXED
 	$(call BUILDSTATUS,TIERDIR_FINISH $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
 endif
 
+# The export tier requires nsinstall, which is built from config. So every
+# subdirectory traversal needs to happen after traversing config.
+ifeq ($(CURRENT_TIER),export)
+$(addsuffix /$(CURRENT_TIER),$(filter-out config,$(CURRENT_DIRS))): config/$(CURRENT_TIER)
+endif
+
 else
 
 # Don't recurse if MAKELEVEL is NO_RECURSE_MAKELEVEL as defined above, but
 # still recurse for externally managed make files (gyp-generated ones).
 ifeq ($(EXTERNALLY_MANAGED_MAKE_FILE)_$(NO_RECURSE_MAKELEVEL),_$(MAKELEVEL))
 
 compile libs export tools::
 
--- a/moz.build
+++ b/moz.build
@@ -1,8 +1,9 @@
+DEFINES['FOO'] = '1'
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 CONFIGURE_SUBST_FILES += ['tools/update-packaging/Makefile']
 
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -289,16 +289,21 @@ class RecursiveMakeBackend(CommonBackend
                 'dist_private',
                 'dist_sdk',
                 'tests',
                 'xpidl',
             ]}
 
         self._traversal = RecursiveMakeTraversal()
 
+        derecurse = self.environment.substs.get('MOZ_PSEUDO_DERECURSE', '').split(',')
+        self._parallel_export = False
+        if derecurse != [''] and not 'no-parallel-export' in derecurse:
+            self._parallel_export = True
+
     def _update_from_avoid_write(self, result):
         existed, updated = result
 
         if not existed:
             self.summary.created_count += 1
         elif updated:
             self.summary.updated_count += 1
         else:
@@ -385,34 +390,41 @@ class RecursiveMakeBackend(CommonBackend
         self._backend_files[obj.srcdir] = backend_file
 
     def _fill_root_mk(self):
         """
         Create two files, root.mk and root-deps.mk, the first containing
         convenience variables, and the other dependency definitions for a
         hopefully proper directory traversal.
         """
-        # Skip static dirs during export traversal
-        def export_filter(current, subdirs):
-            return current, subdirs.parallel, \
-                subdirs.dirs + subdirs.tests + subdirs.tools
-
-        # compile and tools build everything in parallel, but skip precompile.
-        def other_filter(current, subdirs):
-            if current == 'subtiers/precompile':
-                return None, [], []
+        # Traverse directories in parallel, and skip static dirs
+        def parallel_filter(current, subdirs):
             all_subdirs = subdirs.parallel + subdirs.dirs + \
                           subdirs.tests + subdirs.tools
             # subtiers/*_start and subtiers/*_finish, under subtiers/*, are
             # kept sequential. Others are all forced parallel.
             if current.startswith('subtiers/') and all_subdirs and \
                     all_subdirs[0].startswith('subtiers/'):
                 return current, [], all_subdirs
             return current, all_subdirs, []
 
+        # Skip static dirs during export traversal, or build everything in
+        # parallel when enabled.
+        def export_filter(current, subdirs):
+            if self._parallel_export:
+                return parallel_filter(current, subdirs)
+            return current, subdirs.parallel, \
+                subdirs.dirs + subdirs.tests + subdirs.tools
+
+        # compile and tools build everything in parallel, but skip precompile.
+        def other_filter(current, subdirs):
+            if current == 'subtiers/precompile':
+                return None, [], []
+            return parallel_filter(current, subdirs)
+
         # Skip tools dirs during libs traversal
         def libs_filter(current, subdirs):
             if current == 'subtiers/precompile':
                 return None, [], []
             return current, subdirs.parallel, \
                 subdirs.static + subdirs.dirs + subdirs.tests
 
         # compile and tools tiers use the same traversal as export