Bug 1620133 - Allow generated file rules to run in parallel in a given directory. r=firefox-build-system-reviewers,rstewart
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 28 Aug 2020 01:58:48 +0000
changeset 546717 3f6d140abf16111a7ad5a9633a202803bb68e707
parent 546716 8513e2bba101fd4822ba1c02c42462191e13bc97
child 546718 478edcb2dc6e57d6ed4fae1db26d562f794e2e19
push id37736
push userapavel@mozilla.com
push dateFri, 28 Aug 2020 15:31:26 +0000
treeherdermozilla-central@56166cae2e26 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfirefox-build-system-reviewers, rstewart
bugs1620133, 1645986
milestone82.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1620133 - Allow generated file rules to run in parallel in a given directory. r=firefox-build-system-reviewers,rstewart Bug 1645986 solved the problem for most generated files by moving their rules to the top-level, but we're going to add rules that will end up in subdirectories, so we have to solve the same problem again, in the subdirectories. Differential Revision: https://phabricator.services.mozilla.com/D88389
config/config.mk
config/makefiles/rust.mk
python/mozbuild/mozbuild/backend/make.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
--- a/config/config.mk
+++ b/config/config.mk
@@ -6,16 +6,18 @@
 #
 # config.mk
 #
 # Determines the platform and builds the macros needed to load the
 # appropriate platform-specific .mk file, then defines all (most?)
 # of the generic macros.
 #
 
+varize = $(subst -,_,$(subst a,A,$(subst b,B,$(subst c,C,$(subst d,D,$(subst e,E,$(subst f,F,$(subst g,G,$(subst h,H,$(subst i,I,$(subst j,J,$(subst k,K,$(subst l,L,$(subst m,M,$(subst n,N,$(subst o,O,$(subst p,P,$(subst q,Q,$(subst r,R,$(subst s,S,$(subst t,T,$(subst u,U,$(subst v,V,$(subst w,W,$(subst x,X,$(subst y,Y,$(subst z,Z,$1)))))))))))))))))))))))))))
+
 # Define an include-at-most-once flag
 ifdef INCLUDED_CONFIG_MK
 $(error Do not include config.mk twice!)
 endif
 INCLUDED_CONFIG_MK = 1
 
 EXIT_ON_ERROR = set -e; # Shell loops continue past errors without this.
 
@@ -34,16 +36,20 @@ MDDEPDIR := .deps
 
 ifndef EXTERNALLY_MANAGED_MAKE_FILE
 # Import the automatically generated backend file. If this file doesn't exist,
 # the backend hasn't been properly configured. We want this to be a fatal
 # error, hence not using "-include".
 ifndef STANDALONE_MAKEFILE
 GLOBAL_DEPS += backend.mk
 include backend.mk
+
+# Add e.g. `export:: $(EXPORT_TARGETS)` rules. The *_TARGETS variables are defined
+# in backend.mk.
+$(foreach tier,$(RUNNABLE_TIERS),$(eval $(tier):: $($(call varize,$(tier))_TARGETS)))
 endif
 
 endif
 
 space = $(NULL) $(NULL)
 
 # Include defs.mk files that can be found in $(srcdir)/$(DEPTH),
 # $(srcdir)/$(DEPTH-1), $(srcdir)/$(DEPTH-2), etc., and $(srcdir)
--- a/config/makefiles/rust.mk
+++ b/config/makefiles/rust.mk
@@ -206,18 +206,16 @@ HOST_RECIPES := \
 
 # If this is a release build we want rustc to generate one codegen unit per
 # crate. This results in better optimization and less code duplication at the
 # cost of longer compile times.
 ifndef DEVELOPER_OPTIONS
 $(TARGET_RECIPES) $(HOST_RECIPES): RUSTFLAGS += -C codegen-units=1
 endif
 
-cargo_env = $(subst -,_,$(subst a,A,$(subst b,B,$(subst c,C,$(subst d,D,$(subst e,E,$(subst f,F,$(subst g,G,$(subst h,H,$(subst i,I,$(subst j,J,$(subst k,K,$(subst l,L,$(subst m,M,$(subst n,N,$(subst o,O,$(subst p,P,$(subst q,Q,$(subst r,R,$(subst s,S,$(subst t,T,$(subst u,U,$(subst v,V,$(subst w,W,$(subst x,X,$(subst y,Y,$(subst z,Z,$1)))))))))))))))))))))))))))
-
 # We use the + prefix to pass down the jobserver fds to cargo, but we
 # don't use the prefix when make -n is used, so that cargo doesn't run
 # in that case)
 define RUN_CARGO
 $(if $(findstring n,$(filter-out --%, $(MAKEFLAGS))),,+)$(CARGO) $(1) $(cargo_build_flags)
 endef
 
 # This function is intended to be called by:
@@ -230,18 +228,18 @@ endef
 define CARGO_BUILD
 $(call RUN_CARGO,rustc)
 endef
 
 define CARGO_CHECK
 $(call RUN_CARGO,check)
 endef
 
-cargo_host_linker_env_var := CARGO_TARGET_$(call cargo_env,$(RUST_HOST_TARGET))_LINKER
-cargo_linker_env_var := CARGO_TARGET_$(call cargo_env,$(RUST_TARGET))_LINKER
+cargo_host_linker_env_var := CARGO_TARGET_$(call varize,$(RUST_HOST_TARGET))_LINKER
+cargo_linker_env_var := CARGO_TARGET_$(call varize,$(RUST_TARGET))_LINKER
 
 # Cargo needs the same linker flags as the C/C++ compiler,
 # but not the final libraries. Filter those out because they
 # cause problems on macOS 10.7; see bug 1365993 for details.
 # Also, we don't want to pass PGO flags until cargo supports them.
 export MOZ_CARGO_WRAP_LDFLAGS
 export MOZ_CARGO_WRAP_LD
 export MOZ_CARGO_WRAP_HOST_LDFLAGS
--- a/python/mozbuild/mozbuild/backend/make.py
+++ b/python/mozbuild/mozbuild/backend/make.py
@@ -75,19 +75,25 @@ class MakeBackend(CommonBackend):
             # we can skip generated files that are needed during compile,
             # or let the rule run as the result of something depending on
             # it.
             if not (obj.required_before_compile or obj.required_during_compile) or \
                     not self.environment.is_artifact_build:
                 if tier and not needs_AB_rCD:
                     # Android localized resources have special Makefile
                     # handling.
-                    double_colon_tiers = ('export', 'pre-compile', 'libs', 'misc')
-                    ret.append('%s%s %s' % (
-                        tier, '::' if tier in double_colon_tiers else ':', stub_file))
+
+                    # Double-colon tiers via a variable that the backend adds as a dependency
+                    # later. See https://bugzilla.mozilla.org/show_bug.cgi?id=1645986#c0 as
+                    # to why.
+                    if tier in ('export', 'pre-compile', 'libs', 'misc'):
+                        dep = '%s_TARGETS' % tier.replace('-', '_').upper()
+                        ret.append('%s += %s' % (dep, stub_file))
+                    else:
+                        ret.append('%s: %s' % (tier, stub_file))
             for output in outputs:
                 ret.append('%s: %s ;' % (output, stub_file))
                 ret.append('GARBAGE += %s' % output)
             ret.append('GARBAGE += %s' % stub_file)
             ret.append('EXTRA_MDDEPEND_FILES += %s' % dep_file)
 
             ret.append((
                     """{stub}: {script}{inputs}{backend}{force}
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -397,27 +397,27 @@ class TestRecursiveMakeBackend(BackendTe
         """Ensure GENERATED_FILES is handled properly."""
         env = self._consume('generated-files', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
         expected = [
             'include $(topsrcdir)/config/AB_rCD.mk',
-            'pre-compile:: $(MDDEPDIR)/bar.c.stub',
+            'PRE_COMPILE_TARGETS += $(MDDEPDIR)/bar.c.stub',
             'bar.c: $(MDDEPDIR)/bar.c.stub ;',
             'GARBAGE += bar.c',
             'GARBAGE += $(MDDEPDIR)/bar.c.stub',
             'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/bar.c.pp',
             '$(MDDEPDIR)/bar.c.stub: %s/generate-bar.py' % env.topsrcdir,
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,%s/generate-bar.py baz bar.c $(MDDEPDIR)/bar.c.pp $(MDDEPDIR)/bar.c.stub)' % env.topsrcdir,  # noqa
             '@$(TOUCH) $@',
             '',
-            'export:: $(MDDEPDIR)/foo.h.stub',
+            'EXPORT_TARGETS += $(MDDEPDIR)/foo.h.stub',
             'foo.h: $(MDDEPDIR)/foo.h.stub ;',
             'GARBAGE += foo.h',
             'GARBAGE += $(MDDEPDIR)/foo.h.stub',
             'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/foo.h.pp',
             '$(MDDEPDIR)/foo.h.stub: %s/generate-foo.py $(srcdir)/foo-data' % (env.topsrcdir),
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,%s/generate-foo.py main foo.h $(MDDEPDIR)/foo.h.pp $(MDDEPDIR)/foo.h.stub $(srcdir)/foo-data)' % (env.topsrcdir),  # noqa
             '@$(TOUCH) $@',
@@ -431,27 +431,27 @@ class TestRecursiveMakeBackend(BackendTe
         """Ensure GENERATED_FILES with .force is handled properly."""
         env = self._consume('generated-files-force', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
         expected = [
             'include $(topsrcdir)/config/AB_rCD.mk',
-            'pre-compile:: $(MDDEPDIR)/bar.c.stub',
+            'PRE_COMPILE_TARGETS += $(MDDEPDIR)/bar.c.stub',
             'bar.c: $(MDDEPDIR)/bar.c.stub ;',
             'GARBAGE += bar.c',
             'GARBAGE += $(MDDEPDIR)/bar.c.stub',
             'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/bar.c.pp',
             '$(MDDEPDIR)/bar.c.stub: %s/generate-bar.py FORCE' % env.topsrcdir,
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,%s/generate-bar.py baz bar.c $(MDDEPDIR)/bar.c.pp $(MDDEPDIR)/bar.c.stub)' % env.topsrcdir,  # noqa
             '@$(TOUCH) $@',
             '',
-            'pre-compile:: $(MDDEPDIR)/foo.c.stub',
+            'PRE_COMPILE_TARGETS += $(MDDEPDIR)/foo.c.stub',
             'foo.c: $(MDDEPDIR)/foo.c.stub ;',
             'GARBAGE += foo.c',
             'GARBAGE += $(MDDEPDIR)/foo.c.stub',
             'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/foo.c.pp',
             '$(MDDEPDIR)/foo.c.stub: %s/generate-foo.py $(srcdir)/foo-data' % (env.topsrcdir),
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,%s/generate-foo.py main foo.c $(MDDEPDIR)/foo.c.pp $(MDDEPDIR)/foo.c.stub $(srcdir)/foo-data)' % (env.topsrcdir),  # noqa
             '@$(TOUCH) $@',
@@ -465,17 +465,17 @@ class TestRecursiveMakeBackend(BackendTe
         """Ensure LOCALIZED_GENERATED_FILES is handled properly."""
         env = self._consume('localized-generated-files', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
         expected = [
             'include $(topsrcdir)/config/AB_rCD.mk',
-            'misc:: $(MDDEPDIR)/foo.xyz.stub',
+            'MISC_TARGETS += $(MDDEPDIR)/foo.xyz.stub',
             'foo.xyz: $(MDDEPDIR)/foo.xyz.stub ;',
             'GARBAGE += foo.xyz',
             'GARBAGE += $(MDDEPDIR)/foo.xyz.stub',
             'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/foo.xyz.pp',
             '$(MDDEPDIR)/foo.xyz.stub: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir,  # noqa
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main foo.xyz $(MDDEPDIR)/foo.xyz.pp $(MDDEPDIR)/foo.xyz.stub $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir,  # noqa
             '@$(TOUCH) $@',
@@ -493,27 +493,27 @@ class TestRecursiveMakeBackend(BackendTe
         """Ensure LOCALIZED_GENERATED_FILES with .force is handled properly."""
         env = self._consume('localized-generated-files-force', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
         expected = [
             'include $(topsrcdir)/config/AB_rCD.mk',
-            'misc:: $(MDDEPDIR)/foo.xyz.stub',
+            'MISC_TARGETS += $(MDDEPDIR)/foo.xyz.stub',
             'foo.xyz: $(MDDEPDIR)/foo.xyz.stub ;',
             'GARBAGE += foo.xyz',
             'GARBAGE += $(MDDEPDIR)/foo.xyz.stub',
             'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/foo.xyz.pp',
             '$(MDDEPDIR)/foo.xyz.stub: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir,  # noqa
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main foo.xyz $(MDDEPDIR)/foo.xyz.pp $(MDDEPDIR)/foo.xyz.stub $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir,  # noqa
             '@$(TOUCH) $@',
             '',
-            'misc:: $(MDDEPDIR)/abc.xyz.stub',
+            'MISC_TARGETS += $(MDDEPDIR)/abc.xyz.stub',
             'abc.xyz: $(MDDEPDIR)/abc.xyz.stub ;',
             'GARBAGE += abc.xyz',
             'GARBAGE += $(MDDEPDIR)/abc.xyz.stub',
             'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/abc.xyz.pp',
             '$(MDDEPDIR)/abc.xyz.stub: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input FORCE' % env.topsrcdir,  # noqa
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main abc.xyz $(MDDEPDIR)/abc.xyz.pp $(MDDEPDIR)/abc.xyz.stub $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir,  # noqa
             '@$(TOUCH) $@',
@@ -528,17 +528,17 @@ class TestRecursiveMakeBackend(BackendTe
         when {AB_CD} and {AB_rCD} are used."""
         env = self._consume('localized-generated-files-AB_CD', RecursiveMakeBackend)
 
         backend_path = mozpath.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
         expected = [
             'include $(topsrcdir)/config/AB_rCD.mk',
-            'misc:: $(MDDEPDIR)/foo$(AB_CD).xyz.stub',
+            'MISC_TARGETS += $(MDDEPDIR)/foo$(AB_CD).xyz.stub',
             'foo$(AB_CD).xyz: $(MDDEPDIR)/foo$(AB_CD).xyz.stub ;',
             'GARBAGE += foo$(AB_CD).xyz',
             'GARBAGE += $(MDDEPDIR)/foo$(AB_CD).xyz.stub',
             'EXTRA_MDDEPEND_FILES += $(MDDEPDIR)/foo$(AB_CD).xyz.pp',
             '$(MDDEPDIR)/foo$(AB_CD).xyz.stub: %s/generate-foo.py $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input $(if $(IS_LANGUAGE_REPACK),FORCE)' % env.topsrcdir,  # noqa
             '$(REPORT_BUILD)',
             '$(call py_action,file_generate,--locale=$(AB_CD) %s/generate-foo.py main foo$(AB_CD).xyz $(MDDEPDIR)/foo$(AB_CD).xyz.pp $(MDDEPDIR)/foo$(AB_CD).xyz.stub $(call MERGE_FILE,localized-input) $(srcdir)/non-localized-input)' % env.topsrcdir,  # noqa
             '@$(TOUCH) $@',