Bug 899792 - Establish precompile tier and move IPDL and WebIDL build rules; r=glandium
authorGregory Szorc <gps@mozilla.com>
Tue, 30 Jul 2013 17:00:24 -0700
changeset 153138 3192aca82ff316a1ed19e400dfa98f31abe33e25
parent 153137 bb8539a50e37bd508ff41300962963b57c40bd55
child 153139 b16893e6cb289a55e068f9531e441319d35e7f06
push idunknown
push userunknown
push dateunknown
reviewersglandium
bugs899792
milestone25.0a1
Bug 899792 - Establish precompile tier and move IPDL and WebIDL build rules; r=glandium A new build tier is introduced: precompile. The intention is to replace the export subtier with something better and nonrecursive. IPDL and WebIDL code generation have been moved into this new tier and are performed concurrently. This appears to reduce build times due to fewer "single threaded" build actions. The subtier tracking UI has been updated to reflect operations that can occur in parallel. As subtiers are started, they turn yellow. They can then complete in any order. Nonrecursive targets have been added to preserve the IPDL and WebIDL build targets such that developers can type |make| from leaf directories like they have always done.
config/makefiles/nonrecursive.mk
config/makefiles/precompile/Makefile.in
config/makefiles/precompile/moz.build
dom/bindings/Makefile.in
ipc/ipdl/Makefile.in
js/src/config/makefiles/nonrecursive.mk
moz.build
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/mach_commands.py
toolkit/toolkit.mozbuild
--- a/config/makefiles/nonrecursive.mk
+++ b/config/makefiles/nonrecursive.mk
@@ -14,49 +14,55 @@
 # NONRECURSIVE_TARGETS variable lists the make targets that modified. For
 # each target in this list, the NONRECURSIVE_TARGET_<target> variable will
 # contain a list of partial variable names. We will then look in variables
 # named NONRECURSIVE_TARGETS_<target>_<fragment>_* for information describing
 # how to evaluate non-recursive make targets.
 #
 # Targets are defined by the following variables:
 #
-#   FILE - The make file to evaluate.
+#   FILE - The make file to evaluate. This is equivalent to
+#      |make -f <FILE>|
+#   DIRECTORY - The directory whose Makefile to evaluate. This is
+#      equivalent to |make -C <DIRECTORY>|.
 #   TARGETS - Targets to evaluate in that make file.
 #
+# Only 1 of FILE or DIRECTORY may be defined.
+#
 # For example:
 #
 # NONRECURSIVE_TARGETS = export libs
 # NONRECURSIVE_TARGETS_export = headers
 # NONRECURSIVE_TARGETS_export_headers_FILE = /path/to/exports.mk
 # NONRECURSIVE_TARGETS_export_headers_TARGETS = $(DIST)/include/foo.h $(DIST)/include/bar.h
 # NONRECURSIVE_TARGETS_libs = cppsrcs
-# NONRECURSIVE_TARGETS_libs_cppsrcs_FILE = /path/to/compilation.mk
+# NONRECURSIVE_TARGETS_libs_cppsrcs_DIRECTORY = $(DEPTH)/foo
 # NONRECURSIVE_TARGETS_libs_cppsrcs_TARGETS = /path/to/foo.o /path/to/bar.o
 #
 # Will get turned into the following:
 #
 # exports::
-#     $(MAKE) -f /path/to/exports.mk $(DIST)/include/foo.h $(DIST)/include/bar.h
+#     $(MAKE) -C $(DEPTH) -f /path/to/exports.mk $(DIST)/include/foo.h $(DIST)/include/bar.h
 #
 # libs::
-#     $(MAKE) -f /path/to/compilation.mk /path/to/foo.o /path/to/bar.o
+#     $(MAKE) -C $(DEPTH)/foo /path/to/foo.o /path/to/bar.o
 
 ifndef INCLUDED_NONRECURSIVE_MK
 
 define define_nonrecursive_target
 $(1)::
-	cd $$(DEPTH) && $$(MAKE) -f $(2) $(3)
+	$$(MAKE) -C $(or $(4),$$(DEPTH)) $(addprefix -f ,$(3)) $(2)
 endef
 
 $(foreach target,$(NONRECURSIVE_TARGETS), \
     $(foreach entry,$(NONRECURSIVE_TARGETS_$(target)), \
         $(eval $(call define_nonrecursive_target, \
             $(target), \
+            $(NONRECURSIVE_TARGETS_$(target)_$(entry)_TARGETS), \
             $(NONRECURSIVE_TARGETS_$(target)_$(entry)_FILE), \
-            $(NONRECURSIVE_TARGETS_$(target)_$(entry)_TARGETS) \
+            $(NONRECURSIVE_TARGETS_$(target)_$(entry)_DIRECTORY), \
         )) \
     ) \
 )
 
 INCLUDED_NONRECURSIVE_MK := 1
 endif
 
new file mode 100644
--- /dev/null
+++ b/config/makefiles/precompile/Makefile.in
@@ -0,0 +1,34 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This make file defines the precompile tier. This tier effectively fans out
+# to other make files and specialized targets.
+
+DEPTH := @DEPTH@
+topsrcdir := @top_srcdir@
+srcdir := @srcdir@
+
+include $(DEPTH)/config/autoconf.mk
+
+SUPPRESS_DEFAULT_RULES := 1
+
+include $(topsrcdir)/config/rules.mk
+
+define make_subtier_dir
+@echo "BUILDSTATUS SUBTIER_START precompile $(1)"
+$(MAKE) -C $(2) $(3)
+@echo "BUILDSTATUS SUBTIER_FINISH precompile $(1)"
+
+endef
+
+export::
+	@echo "BUILDSTATUS SUBTIERS IPDL WebIDL"
+
+export:: ipdl webidl
+
+ipdl:
+	$(call make_subtier_dir,IPDL,$(DEPTH)/ipc/ipdl,ipdl)
+
+webidl:
+	$(call make_subtier_dir,WebIDL,$(DEPTH)/dom/bindings,webidl)
new file mode 100644
--- /dev/null
+++ b/config/makefiles/precompile/moz.build
@@ -0,0 +1,5 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
--- a/dom/bindings/Makefile.in
+++ b/dom/bindings/Makefile.in
@@ -237,18 +237,26 @@ GARBAGE += \
   $(binding_dependency_trackers) \
   $(NULL)
 
 # Make sure all binding header files are created during the export stage, so we
 # don't have issues with .cpp files being compiled before we've generated the
 # headers they depend on.  This is really only needed for the test files, since
 # the non-test headers are all exported above anyway.  Note that this means that
 # we do all of our codegen during export.
-export:: $(binding_header_files)
+webidl: $(binding_header_files)
+
+.PHONY: webidl
 
 distclean::
 	-$(RM) \
         $(binding_header_files) \
         $(binding_cpp_files) \
         $(all_webidl_files) \
         $(globalgen_targets) \
         ParserResults.pkl \
         $(NULL)
+
+# This is only needed to support |make| from this leaf directory/Makefile.
+NONRECURSIVE_TARGETS := export
+NONRECURSIVE_TARGETS_export := webidl
+NONRECURSIVE_TARGETS_export_webidl_DIRECTORY := .
+NONRECURSIVE_TARGETS_export_webidl_TARGETS := webidl
--- a/ipc/ipdl/Makefile.in
+++ b/ipc/ipdl/Makefile.in
@@ -26,20 +26,28 @@ LOCAL_INCLUDES += -I$(DEPTH)/ipc/ipdl/_i
 
 include $(topsrcdir)/config/config.mk
 include $(topsrcdir)/config/rules.mk
 include $(topsrcdir)/ipc/chromium/chromium-config.mk
 
 
 # NB: the IPDL compiler manages .ipdl-->.h/.cpp dependencies itself,
 # which is why we don't have explicit .h/.cpp targets here
-export:: $(ALL_IPDLSRCS)
+ipdl: $(ALL_IPDLSRCS)
 	$(PYTHON) $(topsrcdir)/config/pythonpath.py \
 	  $(PLY_INCLUDE) \
 	  $(srcdir)/ipdl.py \
 	  --outheaders-dir=_ipdlheaders \
 	  --outcpp-dir=. \
 	  $(IPDLDIRS:%=-I%) \
 	  $^
 
+.PHONY: ipdl
+
 # We #include some things in the dom/plugins/ directory that rely on
 # toolkit libraries.
 CXXFLAGS    += $(TK_CFLAGS)
+
+# This is only needed to support |make| from this leaf directory/Makefile.
+NONRECURSIVE_TARGETS := export
+NONRECURSIVE_TARGETS_export := ipdl
+NONRECURSIVE_TARGETS_export_ipdl_DIRECTORY := .
+NONRECURSIVE_TARGETS_export_ipdl_TARGETS := ipdl
--- a/js/src/config/makefiles/nonrecursive.mk
+++ b/js/src/config/makefiles/nonrecursive.mk
@@ -14,49 +14,55 @@
 # NONRECURSIVE_TARGETS variable lists the make targets that modified. For
 # each target in this list, the NONRECURSIVE_TARGET_<target> variable will
 # contain a list of partial variable names. We will then look in variables
 # named NONRECURSIVE_TARGETS_<target>_<fragment>_* for information describing
 # how to evaluate non-recursive make targets.
 #
 # Targets are defined by the following variables:
 #
-#   FILE - The make file to evaluate.
+#   FILE - The make file to evaluate. This is equivalent to
+#      |make -f <FILE>|
+#   DIRECTORY - The directory whose Makefile to evaluate. This is
+#      equivalent to |make -C <DIRECTORY>|.
 #   TARGETS - Targets to evaluate in that make file.
 #
+# Only 1 of FILE or DIRECTORY may be defined.
+#
 # For example:
 #
 # NONRECURSIVE_TARGETS = export libs
 # NONRECURSIVE_TARGETS_export = headers
 # NONRECURSIVE_TARGETS_export_headers_FILE = /path/to/exports.mk
 # NONRECURSIVE_TARGETS_export_headers_TARGETS = $(DIST)/include/foo.h $(DIST)/include/bar.h
 # NONRECURSIVE_TARGETS_libs = cppsrcs
-# NONRECURSIVE_TARGETS_libs_cppsrcs_FILE = /path/to/compilation.mk
+# NONRECURSIVE_TARGETS_libs_cppsrcs_DIRECTORY = $(DEPTH)/foo
 # NONRECURSIVE_TARGETS_libs_cppsrcs_TARGETS = /path/to/foo.o /path/to/bar.o
 #
 # Will get turned into the following:
 #
 # exports::
-#     $(MAKE) -f /path/to/exports.mk $(DIST)/include/foo.h $(DIST)/include/bar.h
+#     $(MAKE) -C $(DEPTH) -f /path/to/exports.mk $(DIST)/include/foo.h $(DIST)/include/bar.h
 #
 # libs::
-#     $(MAKE) -f /path/to/compilation.mk /path/to/foo.o /path/to/bar.o
+#     $(MAKE) -C $(DEPTH)/foo /path/to/foo.o /path/to/bar.o
 
 ifndef INCLUDED_NONRECURSIVE_MK
 
 define define_nonrecursive_target
 $(1)::
-	cd $$(DEPTH) && $$(MAKE) -f $(2) $(3)
+	$$(MAKE) -C $(or $(4),$$(DEPTH)) $(addprefix -f ,$(3)) $(2)
 endef
 
 $(foreach target,$(NONRECURSIVE_TARGETS), \
     $(foreach entry,$(NONRECURSIVE_TARGETS_$(target)), \
         $(eval $(call define_nonrecursive_target, \
             $(target), \
+            $(NONRECURSIVE_TARGETS_$(target)_$(entry)_TARGETS), \
             $(NONRECURSIVE_TARGETS_$(target)_$(entry)_FILE), \
-            $(NONRECURSIVE_TARGETS_$(target)_$(entry)_TARGETS) \
+            $(NONRECURSIVE_TARGETS_$(target)_$(entry)_DIRECTORY), \
         )) \
     ) \
 )
 
 INCLUDED_NONRECURSIVE_MK := 1
 endif
 
--- a/moz.build
+++ b/moz.build
@@ -21,9 +21,8 @@ if not CONFIG['LIBXUL_SDK']:
     if not CONFIG['MOZ_NATIVE_ZLIB']:
         add_tier_dir('base', ['modules/zlib'])
 
     add_tier_dir('base', ['mozglue', 'memory/mozalloc'])
 
 # Bring in the configuration for the configured application.
 if CONFIG['COMPILE_ENVIRONMENT']:
     include('/' + CONFIG['MOZ_BUILD_APP'] + '/app.mozbuild')
-
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -42,16 +42,18 @@ class BuildMonitor(object):
 
         self.tiers = []
         self.subtiers = []
         self.current_tier = None
         self.current_subtier = None
         self.current_tier_dirs = []
         self.current_tier_static_dirs = []
         self.current_subtier_dirs = []
+        self.current_subtier_started = set()
+        self.current_subtier_finished = set()
         self.current_tier_dir = None
         self.current_tier_dir_index = 0
 
         self.warnings_database = WarningsDatabase()
         if os.path.exists(warnings_path):
             try:
                 self.warnings_database.load_from_file(warnings_path)
             except ValueError:
@@ -102,35 +104,38 @@ class BuildMonitor(object):
             elif action == 'DIRS':
                 self.current_tier_dirs = args
                 update_needed = False
             elif action == 'TIER_START':
                 assert len(args) == 1
                 self.current_tier = args[0]
                 self.current_subtier = None
                 self.current_tier_dirs = []
+                self.current_subtier_started = set()
+                self.current_subtier_finished = set()
                 self.current_tier_dir = None
             elif action == 'TIER_FINISH':
                 assert len(args) == 1
                 assert args[0] == self.current_tier
             elif action == 'SUBTIER_START':
                 assert len(args) == 2
                 tier, subtier = args
                 assert tier == self.current_tier
                 self.current_subtier = subtier
                 if subtier == 'static':
                     self.current_subtier_dirs = self.current_tier_static_dirs
                 else:
                     self.current_subtier_dirs = self.current_tier_dirs
                 self.current_tier_dir_index = 0
+                self.current_subtier_started.add(subtier)
             elif action == 'SUBTIER_FINISH':
                 assert len(args) == 2
                 tier, subtier = args
                 assert tier == self.current_tier
-                assert subtier == self.current_subtier
+                self.current_subtier_finished.add(subtier)
             elif action == 'TIERDIR_START':
                 assert len(args) == 1
                 self.current_tier_dir = args[0]
                 self.current_tier_dir_index += 1
             elif action == 'TIERDIR_FINISH':
                 assert len(args) == 1
                 assert self.current_tier_dir == args[0]
             else:
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -124,24 +124,22 @@ class BuildProgressFooter(object):
             if tier == self._monitor.current_tier:
                 parts.extend([('underline_yellow', tier), ' '])
                 current_encountered = True
             elif not current_encountered:
                 parts.extend([('green', tier), ' '])
             else:
                 parts.extend([tier, ' '])
 
-        current_encountered = False
         parts.extend([('bold', 'SUBTIER'), ':', ' '])
         for subtier in self._monitor.subtiers:
-            if subtier == self._monitor.current_subtier:
+            if subtier in self._monitor.current_subtier_finished:
+                parts.extend([('green', subtier), ' '])
+            elif subtier in self._monitor.current_subtier_started:
                 parts.extend([('underline_yellow', subtier), ' '])
-                current_encountered = True
-            elif not current_encountered:
-                parts.extend([('green', subtier), ' '])
             else:
                 parts.extend([subtier, ' '])
 
         if self._monitor.current_subtier_dirs and self._monitor.current_tier_dir:
             parts.extend([
                 ('bold', 'DIRECTORIES'), ': ',
                 '%02d' % self._monitor.current_tier_dir_index,
                 '/',
--- a/toolkit/toolkit.mozbuild
+++ b/toolkit/toolkit.mozbuild
@@ -4,16 +4,18 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 if CONFIG['LIBXUL_SDK']:
     error('toolkit.mozbuild is not compatible with --enable-libxul-sdk=')
 
 if not CONFIG['MOZ_NATIVE_NSPR']:
     add_tier_dir('nspr', 'config/nspr')
 
+add_tier_dir('precompile', 'config/makefiles/precompile')
+
 if not CONFIG['MOZ_NATIVE_SQLITE']:
     add_tier_dir('nss', 'db/sqlite3/src')
 
 if not CONFIG['MOZ_NATIVE_NSS']:
     add_tier_dir('nss', 'security/build')
 
 include('/config/js/js.mozbuild')