Bug 915648 - Parallelize make export. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 20 Sep 2013 13:30:17 +0900
changeset 161928 9cc7634555c3e7a5d206cdd90d08661f6504853a
parent 161927 0d19104a7002076c73555a1b860d2c1ba4bcca73
child 161929 9eba1b4d798f74a5a0046c54cc12cbbfbc69b1bc
push id3066
push userakeybl@mozilla.com
push dateMon, 09 Dec 2013 19:58:46 +0000
treeherdermozilla-beta@a31a0dce83aa [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs915648
milestone27.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 915648 - Parallelize make export. r=gps
Makefile.in
config/recurse.mk
js/src/config/recurse.mk
moz.build
python/mozbuild/mozbuild/backend/recursivemake.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -212,8 +212,15 @@ endif
 js/src/Makefile: subsrcdir := js/src
 
 ifdef ENABLE_TESTS
 # Incorporate static tier directories into tests. This should be incorporated
 # into moz.build files someday.
 check::
 	$(call SUBMAKE,$@,js/src)
 endif
+
+ifdef MOZ_PSEUDO_DERECURSE
+# Interdependencies for parallel export.
+js/xpconnect/src/export: config/makefiles/precompile/export
+accessible/src/xpcom/export: config/makefiles/precompile/export
+js/src/export: mfbt/export
+endif
--- a/config/recurse.mk
+++ b/config/recurse.mk
@@ -17,17 +17,18 @@ endif
 #     Entering qux
 #
 # Pseudo derecurse transforms the above into:
 #   make -C foo
 #   make -C foo/bar
 #   make -C foo/baz
 #   make -C qux
 
-ifeq (1_.,$(MOZ_PSEUDO_DERECURSE)_$(DEPTH))
+# MOZ_PSEUDO_DERECURSE can have values other than 1.
+ifeq (1_.,$(if $(MOZ_PSEUDO_DERECURSE),1)_$(DEPTH))
 
 include root.mk
 
 # Disable build status for mach in top directories without TIERS.
 # In practice this disables it when recursing under js/src, which confuses mach.
 ifndef TIERS
 BUILDSTATUS =
 endif
@@ -83,16 +84,22 @@ CURRENT_DIRS := $(or $($(CURRENT_TIER)_d
 ifdef BUG_915535_FIXED
 	$(call BUILDSTATUS,TIERDIR_START $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
 endif
 	+@$(MAKE) -C $* $(if $(filter $*,$(tier_$(subtier_of_$(subst /,_,$*))_staticdirs)),,$(CURRENT_TIER))
 ifdef BUG_915535_FIXED
 	$(call BUILDSTATUS,TIERDIR_FINISH $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
 endif
 
+# The export tier requires nsinstall, which is built from config. So every
+# subdirectory traversal needs to happen after traversing config.
+ifeq ($(CURRENT_TIER),export)
+$(addsuffix /$(CURRENT_TIER),$(filter-out config,$(CURRENT_DIRS))): config/$(CURRENT_TIER)
+endif
+
 else
 
 # Don't recurse if MAKELEVEL is NO_RECURSE_MAKELEVEL as defined above, but
 # still recurse for externally managed make files (gyp-generated ones).
 ifeq ($(EXTERNALLY_MANAGED_MAKE_FILE)_$(NO_RECURSE_MAKELEVEL),_$(MAKELEVEL))
 
 compile libs export tools::
 
--- a/js/src/config/recurse.mk
+++ b/js/src/config/recurse.mk
@@ -83,16 +83,22 @@ CURRENT_DIRS := $(or $($(CURRENT_TIER)_d
 ifdef BUG_915535_FIXED
 	$(call BUILDSTATUS,TIERDIR_START $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
 endif
 	+@$(MAKE) -C $* $(if $(filter $*,$(tier_$(subtier_of_$(subst /,_,$*))_staticdirs)),,$(CURRENT_TIER))
 ifdef BUG_915535_FIXED
 	$(call BUILDSTATUS,TIERDIR_FINISH $(CURRENT_TIER) $(subtier_of_$(subst /,_,$*)) $*)
 endif
 
+# The export tier requires nsinstall, which is built from config. So every
+# subdirectory traversal needs to happen after traversing config.
+ifeq ($(CURRENT_TIER),export)
+$(addsuffix /$(CURRENT_TIER),$(filter-out config,$(CURRENT_DIRS))): config/$(CURRENT_TIER)
+endif
+
 else
 
 # Don't recurse if MAKELEVEL is NO_RECURSE_MAKELEVEL as defined above, but
 # still recurse for externally managed make files (gyp-generated ones).
 ifeq ($(EXTERNALLY_MANAGED_MAKE_FILE)_$(NO_RECURSE_MAKELEVEL),_$(MAKELEVEL))
 
 compile libs export tools::
 
--- a/moz.build
+++ b/moz.build
@@ -1,8 +1,9 @@
+DEFINES['FOO'] = '1'
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 CONFIGURE_SUBST_FILES += ['tools/update-packaging/Makefile']
 
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -289,16 +289,21 @@ class RecursiveMakeBackend(CommonBackend
                 'dist_private',
                 'dist_sdk',
                 'tests',
                 'xpidl',
             ]}
 
         self._traversal = RecursiveMakeTraversal()
 
+        derecurse = self.environment.substs.get('MOZ_PSEUDO_DERECURSE', '').split(',')
+        self._parallel_export = False
+        if derecurse != [''] and not 'no-parallel-export' in derecurse:
+            self._parallel_export = True
+
     def _update_from_avoid_write(self, result):
         existed, updated = result
 
         if not existed:
             self.summary.created_count += 1
         elif updated:
             self.summary.updated_count += 1
         else:
@@ -385,34 +390,41 @@ class RecursiveMakeBackend(CommonBackend
         self._backend_files[obj.srcdir] = backend_file
 
     def _fill_root_mk(self):
         """
         Create two files, root.mk and root-deps.mk, the first containing
         convenience variables, and the other dependency definitions for a
         hopefully proper directory traversal.
         """
-        # Skip static dirs during export traversal
-        def export_filter(current, subdirs):
-            return current, subdirs.parallel, \
-                subdirs.dirs + subdirs.tests + subdirs.tools
-
-        # compile and tools build everything in parallel, but skip precompile.
-        def other_filter(current, subdirs):
-            if current == 'subtiers/precompile':
-                return None, [], []
+        # Traverse directories in parallel, and skip static dirs
+        def parallel_filter(current, subdirs):
             all_subdirs = subdirs.parallel + subdirs.dirs + \
                           subdirs.tests + subdirs.tools
             # subtiers/*_start and subtiers/*_finish, under subtiers/*, are
             # kept sequential. Others are all forced parallel.
             if current.startswith('subtiers/') and all_subdirs and \
                     all_subdirs[0].startswith('subtiers/'):
                 return current, [], all_subdirs
             return current, all_subdirs, []
 
+        # Skip static dirs during export traversal, or build everything in
+        # parallel when enabled.
+        def export_filter(current, subdirs):
+            if self._parallel_export:
+                return parallel_filter(current, subdirs)
+            return current, subdirs.parallel, \
+                subdirs.dirs + subdirs.tests + subdirs.tools
+
+        # compile and tools build everything in parallel, but skip precompile.
+        def other_filter(current, subdirs):
+            if current == 'subtiers/precompile':
+                return None, [], []
+            return parallel_filter(current, subdirs)
+
         # Skip tools dirs during libs traversal
         def libs_filter(current, subdirs):
             if current == 'subtiers/precompile':
                 return None, [], []
             return current, subdirs.parallel, \
                 subdirs.static + subdirs.dirs + subdirs.tests
 
         # compile and tools tiers use the same traversal as export