Bug 899792 - Establish precompile tier and move IPDL and WebIDL build rules; r=glandium
A new build tier is introduced: precompile. The intention is to replace
the export subtier with something better and nonrecursive. IPDL and
WebIDL code generation have been moved into this new tier and are
performed concurrently. This appears to reduce build times due to fewer
"single threaded" build actions.
The subtier tracking UI has been updated to reflect operations that can
occur in parallel. As subtiers are started, they turn yellow. They can
then complete in any order.
Nonrecursive targets have been added to preserve the IPDL and WebIDL
build targets such that developers can type |make| from leaf directories
like they have always done.
--- a/config/makefiles/nonrecursive.mk
+++ b/config/makefiles/nonrecursive.mk
@@ -14,49 +14,55 @@
# NONRECURSIVE_TARGETS variable lists the make targets that modified. For
# each target in this list, the NONRECURSIVE_TARGET_<target> variable will
# contain a list of partial variable names. We will then look in variables
# named NONRECURSIVE_TARGETS_<target>_<fragment>_* for information describing
# how to evaluate non-recursive make targets.
#
# Targets are defined by the following variables:
#
-# FILE - The make file to evaluate.
+# FILE - The make file to evaluate. This is equivalent to
+# |make -f <FILE>|
+# DIRECTORY - The directory whose Makefile to evaluate. This is
+# equivalent to |make -C <DIRECTORY>|.
# TARGETS - Targets to evaluate in that make file.
#
+# Only 1 of FILE or DIRECTORY may be defined.
+#
# For example:
#
# NONRECURSIVE_TARGETS = export libs
# NONRECURSIVE_TARGETS_export = headers
# NONRECURSIVE_TARGETS_export_headers_FILE = /path/to/exports.mk
# NONRECURSIVE_TARGETS_export_headers_TARGETS = $(DIST)/include/foo.h $(DIST)/include/bar.h
# NONRECURSIVE_TARGETS_libs = cppsrcs
-# NONRECURSIVE_TARGETS_libs_cppsrcs_FILE = /path/to/compilation.mk
+# NONRECURSIVE_TARGETS_libs_cppsrcs_DIRECTORY = $(DEPTH)/foo
# NONRECURSIVE_TARGETS_libs_cppsrcs_TARGETS = /path/to/foo.o /path/to/bar.o
#
# Will get turned into the following:
#
# exports::
-# $(MAKE) -f /path/to/exports.mk $(DIST)/include/foo.h $(DIST)/include/bar.h
+# $(MAKE) -C $(DEPTH) -f /path/to/exports.mk $(DIST)/include/foo.h $(DIST)/include/bar.h
#
# libs::
-# $(MAKE) -f /path/to/compilation.mk /path/to/foo.o /path/to/bar.o
+# $(MAKE) -C $(DEPTH)/foo /path/to/foo.o /path/to/bar.o
ifndef INCLUDED_NONRECURSIVE_MK
define define_nonrecursive_target
$(1)::
- cd $$(DEPTH) && $$(MAKE) -f $(2) $(3)
+ $$(MAKE) -C $(or $(4),$$(DEPTH)) $(addprefix -f ,$(3)) $(2)
endef
$(foreach target,$(NONRECURSIVE_TARGETS), \
$(foreach entry,$(NONRECURSIVE_TARGETS_$(target)), \
$(eval $(call define_nonrecursive_target, \
$(target), \
+ $(NONRECURSIVE_TARGETS_$(target)_$(entry)_TARGETS), \
$(NONRECURSIVE_TARGETS_$(target)_$(entry)_FILE), \
- $(NONRECURSIVE_TARGETS_$(target)_$(entry)_TARGETS) \
+ $(NONRECURSIVE_TARGETS_$(target)_$(entry)_DIRECTORY), \
)) \
) \
)
INCLUDED_NONRECURSIVE_MK := 1
endif
new file mode 100644
--- /dev/null
+++ b/config/makefiles/precompile/Makefile.in
@@ -0,0 +1,34 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This make file defines the precompile tier. This tier effectively fans out
+# to other make files and specialized targets.
+
+DEPTH := @DEPTH@
+topsrcdir := @top_srcdir@
+srcdir := @srcdir@
+
+include $(DEPTH)/config/autoconf.mk
+
+SUPPRESS_DEFAULT_RULES := 1
+
+include $(topsrcdir)/config/rules.mk
+
+define make_subtier_dir
+@echo "BUILDSTATUS SUBTIER_START precompile $(1)"
+$(MAKE) -C $(2) $(3)
+@echo "BUILDSTATUS SUBTIER_FINISH precompile $(1)"
+
+endef
+
+export::
+ @echo "BUILDSTATUS SUBTIERS IPDL WebIDL"
+
+export:: ipdl webidl
+
+ipdl:
+ $(call make_subtier_dir,IPDL,$(DEPTH)/ipc/ipdl,ipdl)
+
+webidl:
+ $(call make_subtier_dir,WebIDL,$(DEPTH)/dom/bindings,webidl)
new file mode 100644
--- /dev/null
+++ b/config/makefiles/precompile/moz.build
@@ -0,0 +1,5 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
--- a/dom/bindings/Makefile.in
+++ b/dom/bindings/Makefile.in
@@ -237,18 +237,24 @@ GARBAGE += \
$(binding_dependency_trackers) \
$(NULL)
# Make sure all binding header files are created during the export stage, so we
# don't have issues with .cpp files being compiled before we've generated the
# headers they depend on. This is really only needed for the test files, since
# the non-test headers are all exported above anyway. Note that this means that
# we do all of our codegen during export.
-export:: $(binding_header_files)
+webidl: $(binding_header_files)
distclean::
-$(RM) \
$(binding_header_files) \
$(binding_cpp_files) \
$(all_webidl_files) \
$(globalgen_targets) \
ParserResults.pkl \
$(NULL)
+
+# This is only needed to support |make| from this leaf directory/Makefile.
+NONRECURSIVE_TARGETS := export
+NONRECURSIVE_TARGETS_export := webidl
+NONRECURSIVE_TARGETS_export_webidl_DIRECTORY := .
+NONRECURSIVE_TARGETS_export_webidl_TARGETS := webidl
--- a/ipc/ipdl/Makefile.in
+++ b/ipc/ipdl/Makefile.in
@@ -26,20 +26,26 @@ LOCAL_INCLUDES += -I$(DEPTH)/ipc/ipdl/_i
include $(topsrcdir)/config/config.mk
include $(topsrcdir)/config/rules.mk
include $(topsrcdir)/ipc/chromium/chromium-config.mk
# NB: the IPDL compiler manages .ipdl-->.h/.cpp dependencies itself,
# which is why we don't have explicit .h/.cpp targets here
-export:: $(ALL_IPDLSRCS)
+ipdl: $(ALL_IPDLSRCS)
$(PYTHON) $(topsrcdir)/config/pythonpath.py \
$(PLY_INCLUDE) \
$(srcdir)/ipdl.py \
--outheaders-dir=_ipdlheaders \
--outcpp-dir=. \
$(IPDLDIRS:%=-I%) \
$^
# We #include some things in the dom/plugins/ directory that rely on
# toolkit libraries.
CXXFLAGS += $(TK_CFLAGS)
+
+# This is only needed to support |make| from this leaf directory/Makefile.
+NONRECURSIVE_TARGETS := export
+NONRECURSIVE_TARGETS_export := ipdl
+NONRECURSIVE_TARGETS_export_ipdl_DIRECTORY := .
+NONRECURSIVE_TARGETS_export_ipdl_TARGETS := ipdl
--- a/js/src/config/makefiles/nonrecursive.mk
+++ b/js/src/config/makefiles/nonrecursive.mk
@@ -14,49 +14,55 @@
# NONRECURSIVE_TARGETS variable lists the make targets that modified. For
# each target in this list, the NONRECURSIVE_TARGET_<target> variable will
# contain a list of partial variable names. We will then look in variables
# named NONRECURSIVE_TARGETS_<target>_<fragment>_* for information describing
# how to evaluate non-recursive make targets.
#
# Targets are defined by the following variables:
#
-# FILE - The make file to evaluate.
+# FILE - The make file to evaluate. This is equivalent to
+# |make -f <FILE>|
+# DIRECTORY - The directory whose Makefile to evaluate. This is
+# equivalent to |make -C <DIRECTORY>|.
# TARGETS - Targets to evaluate in that make file.
#
+# Only 1 of FILE or DIRECTORY may be defined.
+#
# For example:
#
# NONRECURSIVE_TARGETS = export libs
# NONRECURSIVE_TARGETS_export = headers
# NONRECURSIVE_TARGETS_export_headers_FILE = /path/to/exports.mk
# NONRECURSIVE_TARGETS_export_headers_TARGETS = $(DIST)/include/foo.h $(DIST)/include/bar.h
# NONRECURSIVE_TARGETS_libs = cppsrcs
-# NONRECURSIVE_TARGETS_libs_cppsrcs_FILE = /path/to/compilation.mk
+# NONRECURSIVE_TARGETS_libs_cppsrcs_DIRECTORY = $(DEPTH)/foo
# NONRECURSIVE_TARGETS_libs_cppsrcs_TARGETS = /path/to/foo.o /path/to/bar.o
#
# Will get turned into the following:
#
# exports::
-# $(MAKE) -f /path/to/exports.mk $(DIST)/include/foo.h $(DIST)/include/bar.h
+# $(MAKE) -C $(DEPTH) -f /path/to/exports.mk $(DIST)/include/foo.h $(DIST)/include/bar.h
#
# libs::
-# $(MAKE) -f /path/to/compilation.mk /path/to/foo.o /path/to/bar.o
+# $(MAKE) -C $(DEPTH)/foo /path/to/foo.o /path/to/bar.o
ifndef INCLUDED_NONRECURSIVE_MK
define define_nonrecursive_target
$(1)::
- cd $$(DEPTH) && $$(MAKE) -f $(2) $(3)
+ $$(MAKE) -C $(or $(4),$$(DEPTH)) $(addprefix -f ,$(3)) $(2)
endef
$(foreach target,$(NONRECURSIVE_TARGETS), \
$(foreach entry,$(NONRECURSIVE_TARGETS_$(target)), \
$(eval $(call define_nonrecursive_target, \
$(target), \
+ $(NONRECURSIVE_TARGETS_$(target)_$(entry)_TARGETS), \
$(NONRECURSIVE_TARGETS_$(target)_$(entry)_FILE), \
- $(NONRECURSIVE_TARGETS_$(target)_$(entry)_TARGETS) \
+ $(NONRECURSIVE_TARGETS_$(target)_$(entry)_DIRECTORY), \
)) \
) \
)
INCLUDED_NONRECURSIVE_MK := 1
endif
--- a/moz.build
+++ b/moz.build
@@ -21,9 +21,8 @@ if not CONFIG['LIBXUL_SDK']:
if not CONFIG['MOZ_NATIVE_ZLIB']:
add_tier_dir('base', ['modules/zlib'])
add_tier_dir('base', ['mozglue', 'memory/mozalloc'])
# Bring in the configuration for the configured application.
if CONFIG['COMPILE_ENVIRONMENT']:
include('/' + CONFIG['MOZ_BUILD_APP'] + '/app.mozbuild')
-
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -42,16 +42,18 @@ class BuildMonitor(object):
self.tiers = []
self.subtiers = []
self.current_tier = None
self.current_subtier = None
self.current_tier_dirs = []
self.current_tier_static_dirs = []
self.current_subtier_dirs = []
+ self.current_subtier_started = set()
+ self.current_subtier_finished = set()
self.current_tier_dir = None
self.current_tier_dir_index = 0
self.warnings_database = WarningsDatabase()
if os.path.exists(warnings_path):
try:
self.warnings_database.load_from_file(warnings_path)
except ValueError:
@@ -102,35 +104,38 @@ class BuildMonitor(object):
elif action == 'DIRS':
self.current_tier_dirs = args
update_needed = False
elif action == 'TIER_START':
assert len(args) == 1
self.current_tier = args[0]
self.current_subtier = None
self.current_tier_dirs = []
+ self.current_subtier_started = set()
+ self.current_subtier_finished = set()
self.current_tier_dir = None
elif action == 'TIER_FINISH':
assert len(args) == 1
assert args[0] == self.current_tier
elif action == 'SUBTIER_START':
assert len(args) == 2
tier, subtier = args
assert tier == self.current_tier
self.current_subtier = subtier
if subtier == 'static':
self.current_subtier_dirs = self.current_tier_static_dirs
else:
self.current_subtier_dirs = self.current_tier_dirs
self.current_tier_dir_index = 0
+ self.current_subtier_started.add(subtier)
elif action == 'SUBTIER_FINISH':
assert len(args) == 2
tier, subtier = args
assert tier == self.current_tier
- assert subtier == self.current_subtier
+ self.current_subtier_finished.add(subtier)
elif action == 'TIERDIR_START':
assert len(args) == 1
self.current_tier_dir = args[0]
self.current_tier_dir_index += 1
elif action == 'TIERDIR_FINISH':
assert len(args) == 1
assert self.current_tier_dir == args[0]
else:
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -124,24 +124,22 @@ class BuildProgressFooter(object):
if tier == self._monitor.current_tier:
parts.extend([('underline_yellow', tier), ' '])
current_encountered = True
elif not current_encountered:
parts.extend([('green', tier), ' '])
else:
parts.extend([tier, ' '])
- current_encountered = False
parts.extend([('bold', 'SUBTIER'), ':', ' '])
for subtier in self._monitor.subtiers:
- if subtier == self._monitor.current_subtier:
+ if subtier in self._monitor.current_subtier_finished:
+ parts.extend([('green', subtier), ' '])
+ elif subtier in self._monitor.current_subtier_started:
parts.extend([('underline_yellow', subtier), ' '])
- current_encountered = True
- elif not current_encountered:
- parts.extend([('green', subtier), ' '])
else:
parts.extend([subtier, ' '])
if self._monitor.current_subtier_dirs and self._monitor.current_tier_dir:
parts.extend([
('bold', 'DIRECTORIES'), ': ',
'%02d' % self._monitor.current_tier_dir_index,
'/',
--- a/toolkit/toolkit.mozbuild
+++ b/toolkit/toolkit.mozbuild
@@ -4,16 +4,18 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
if CONFIG['LIBXUL_SDK']:
error('toolkit.mozbuild is not compatible with --enable-libxul-sdk=')
if not CONFIG['MOZ_NATIVE_NSPR']:
add_tier_dir('nspr', 'config/nspr')
+add_tier_dir('precompile', 'config/makefiles/precompile')
+
if not CONFIG['MOZ_NATIVE_SQLITE']:
add_tier_dir('nss', 'db/sqlite3/src')
if not CONFIG['MOZ_NATIVE_NSS']:
add_tier_dir('nss', 'security/build')
include('/config/js/js.mozbuild')
