Bug 877308 - Change when build backend update check it performed; r=glandium
authorGregory Szorc <gps@mozilla.com>
Tue, 29 Oct 2013 14:53:52 -0700
changeset 167474 9d0bf12c1d1d
parent 167473 77570f18be44
child 167475 636620b3af0a
push id3224
push userlsblakk@mozilla.com
push dateTue, 04 Feb 2014 01:06:49 +0000
treeherdermozilla-beta@60c04d0987f1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs877308
milestone28.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 877308 - Change when build backend update check it performed; r=glandium Before, we checked if config.status was stale in any entrant Makefile (top level or child directory). This had undesirable side-effects for partial tree builds, notably that if the build backend was out of date, the current Makefile was invalidated. With this patch, we only regenerate the build config automatically in full/toplevel builds. If an outdated build config is detected on partial tree builds, we error. The impact of this is mitigated by having mach build automatically ensure the build config is current.
Makefile.in
config/rules.mk
js/src/config/rules.mk
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/mach_commands.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -55,16 +55,34 @@ CLOBBER: $(topsrcdir)/CLOBBER
 
 config.status: $(topsrcdir)/configure
 	@echo "STOP!  configure has changed and needs to be run in this build directory."
 	@echo "Please rerun configure."
 	@echo "To ignore this message, touch 'config.status' in the build directory,"
 	@echo "but your build might not succeed."
 	@exit 1
 
+# Regenerate the build backend if it is out of date. We only have this rule in
+# this main make file because having it in rules.mk and applied to partial tree
+# builds resulted in a world of hurt. Gory details are in bug 877308.
+#
+# The mach build driver will ensure the backend is up to date for partial tree
+# builds. This cleanly avoids most of the pain.
+
+backend.RecursiveMakeBackend:
+	@echo "Build configuration changed. Regenerating backend."
+	./config.status
+
+Makefile: backend.RecursiveMakeBackend
+	@$(TOUCH) $@
+
+include backend.RecursiveMakeBackend.pp
+
+default:: backend.RecursiveMakeBackend
+
 ifndef LIBXUL_SDK
 .PHONY: js-config-status
 js-config-status:
 	$(call SUBMAKE,backend.RecursiveMakeBackend,js/src,1)
 endif
 
 install_manifests := bin idl include public private sdk
 install_manifest_depends = \
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -375,24 +375,19 @@ ifdef MACH
 ifndef NO_BUILDSTATUS_MESSAGES
 define BUILDSTATUS
 @echo "BUILDSTATUS $1"
 
 endef
 endif
 endif
 
-# Static directories are largely independent of our build system. But, they
-# could share the same build mechanism (like moz.build files). We need to
-# prevent leaking of our backend state to these independent build systems. This
-# is why MOZBUILD_BACKEND_CHECKED isn't exported to make invocations for static
-# directories.
 define SUBMAKE # $(call SUBMAKE,target,directory,static)
 +@$(UPDATE_TITLE)
-+$(if $(3), MOZBUILD_BACKEND_CHECKED=,) $(MAKE) $(if $(2),-C $(2)) $(1)
++$(MAKE) $(if $(2),-C $(2)) $(1)
 
 endef # The extra line is important here! don't delete it
 
 define TIER_DIR_SUBMAKE
 $(call BUILDSTATUS,TIERDIR_START  $(1) $(2) $(3))
 $(call SUBMAKE,$(4),$(3),$(5))
 $(call BUILDSTATUS,TIERDIR_FINISH $(1) $(2) $(3))
 
@@ -585,42 +580,33 @@ endif
 
 ifeq (,$(CROSS_COMPILE))
 HOST_OUTOPTION = $(OUTOPTION)
 else
 HOST_OUTOPTION = -o # eol
 endif
 ################################################################################
 
-# Regenerate the build backend if it is out of date. We only check this once
-# per traversal, hence the ifdef and the export. This rule needs to come before
-# other rules for the default target or else it may not run in time.
+# Ensure the build config is up to date. This is done automatically when builds
+# are performed through |mach build|. The check here is to catch people not
+# using mach. If we ever enforce builds through mach, this code can be removed.
 ifndef MOZBUILD_BACKEND_CHECKED
-
-# Since Makefile is listed as a global dependency, this has the
-# unfortunate side-effect of invalidating all targets if it is executed.
-# So e.g. if you are in /dom/bindings and /foo/moz.build changes,
-# /dom/bindings will get invalidated. The upside is if the current
-# Makefile/backend.mk is updated as a result of backend regeneration, we
-# actually pick up the changes. This should reduce the amount of
-# required clobbers and is thus the lesser evil.
-Makefile: $(DEPTH)/backend.RecursiveMakeBackend
-	@$(TOUCH) $@
-
+ifndef MACH
+ifndef TOPLEVEL_BUILD
 $(DEPTH)/backend.RecursiveMakeBackend:
-	@echo "Build configuration changed. Regenerating backend."
-	@cd $(DEPTH) && $(PYTHON) ./config.status
-	@$(TOUCH) $@
+	$(error Build configuration changed. Build with |mach build| or run |mach build-backend| to regenerate build config)
 
 include $(DEPTH)/backend.RecursiveMakeBackend.pp
 
 default:: $(DEPTH)/backend.RecursiveMakeBackend
 
 export MOZBUILD_BACKEND_CHECKED=1
 endif
+endif
+endif
 
 # The root makefile doesn't want to do a plain export/libs, because
 # of the tiers and because of libxul. Suppress the default rules in favor
 # of something else. Makefiles which use this var *must* provide a sensible
 # default rule before including rules.mk
 ifndef SUPPRESS_DEFAULT_RULES
 default all::
 	$(MAKE) export
--- a/js/src/config/rules.mk
+++ b/js/src/config/rules.mk
@@ -375,24 +375,19 @@ ifdef MACH
 ifndef NO_BUILDSTATUS_MESSAGES
 define BUILDSTATUS
 @echo "BUILDSTATUS $1"
 
 endef
 endif
 endif
 
-# Static directories are largely independent of our build system. But, they
-# could share the same build mechanism (like moz.build files). We need to
-# prevent leaking of our backend state to these independent build systems. This
-# is why MOZBUILD_BACKEND_CHECKED isn't exported to make invocations for static
-# directories.
 define SUBMAKE # $(call SUBMAKE,target,directory,static)
 +@$(UPDATE_TITLE)
-+$(if $(3), MOZBUILD_BACKEND_CHECKED=,) $(MAKE) $(if $(2),-C $(2)) $(1)
++$(MAKE) $(if $(2),-C $(2)) $(1)
 
 endef # The extra line is important here! don't delete it
 
 define TIER_DIR_SUBMAKE
 $(call BUILDSTATUS,TIERDIR_START  $(1) $(2) $(3))
 $(call SUBMAKE,$(4),$(3),$(5))
 $(call BUILDSTATUS,TIERDIR_FINISH $(1) $(2) $(3))
 
@@ -585,42 +580,33 @@ endif
 
 ifeq (,$(CROSS_COMPILE))
 HOST_OUTOPTION = $(OUTOPTION)
 else
 HOST_OUTOPTION = -o # eol
 endif
 ################################################################################
 
-# Regenerate the build backend if it is out of date. We only check this once
-# per traversal, hence the ifdef and the export. This rule needs to come before
-# other rules for the default target or else it may not run in time.
+# Ensure the build config is up to date. This is done automatically when builds
+# are performed through |mach build|. The check here is to catch people not
+# using mach. If we ever enforce builds through mach, this code can be removed.
 ifndef MOZBUILD_BACKEND_CHECKED
-
-# Since Makefile is listed as a global dependency, this has the
-# unfortunate side-effect of invalidating all targets if it is executed.
-# So e.g. if you are in /dom/bindings and /foo/moz.build changes,
-# /dom/bindings will get invalidated. The upside is if the current
-# Makefile/backend.mk is updated as a result of backend regeneration, we
-# actually pick up the changes. This should reduce the amount of
-# required clobbers and is thus the lesser evil.
-Makefile: $(DEPTH)/backend.RecursiveMakeBackend
-	@$(TOUCH) $@
-
+ifndef MACH
+ifndef TOPLEVEL_BUILD
 $(DEPTH)/backend.RecursiveMakeBackend:
-	@echo "Build configuration changed. Regenerating backend."
-	@cd $(DEPTH) && $(PYTHON) ./config.status
-	@$(TOUCH) $@
+	$(error Build configuration changed. Build with |mach build| or run |mach build-backend| to regenerate build config)
 
 include $(DEPTH)/backend.RecursiveMakeBackend.pp
 
 default:: $(DEPTH)/backend.RecursiveMakeBackend
 
 export MOZBUILD_BACKEND_CHECKED=1
 endif
+endif
+endif
 
 # The root makefile doesn't want to do a plain export/libs, because
 # of the tiers and because of libxul. Suppress the default rules in favor
 # of something else. Makefiles which use this var *must* provide a sensible
 # default rule before including rules.mk
 ifndef SUPPRESS_DEFAULT_RULES
 default all::
 	$(MAKE) export
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -730,16 +730,20 @@ class RecursiveMakeBackend(CommonBackend
         # rules.mk. If they are different, the dependencies don't get pulled in
         # properly.
         with self._write_file('%s.pp' % self._backend_output_list_file) as backend_deps:
             backend_deps.write('$(DEPTH)/backend.%s: %s\n' %
                 (self.__class__.__name__, ' '.join(inputs)))
             for path in inputs:
                 backend_deps.write('%s:\n' % path)
 
+        with open(self._backend_output_list_file, 'a'):
+            pass
+        os.utime(self._backend_output_list_file, None)
+
         # Make the master test manifest files.
         for flavor, t in self._test_manifests.items():
             install_prefix, manifests = t
             manifest_stem = os.path.join(install_prefix, '%s.ini' % flavor)
             self._write_master_test_manifest(os.path.join(
                 self.environment.topobjdir, '_tests', manifest_stem),
                 manifests)
 
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -350,16 +350,26 @@ class Build(MachCommandBase):
                     new_pairs = list(add_extra_dependencies(target_pairs, dm))
                     self.log(logging.DEBUG, 'dumbmake',
                              {'target_pairs': target_pairs,
                               'new_pairs': new_pairs},
                              'Added extra dependencies: will build {new_pairs} ' +
                              'instead of {target_pairs}.')
                     target_pairs = new_pairs
 
+                # Ensure build backend is up to date. The alternative is to
+                # have rules in the invoked Makefile to rebuild the build
+                # backend. But that involves make reinvoking itself and there
+                # are undesired side-effects of this. See bug 877308 for a
+                # comprehensive history lesson.
+                self._run_make(directory=self.topobjdir,
+                    target='backend.RecursiveMakeBackend',
+                    force_pymake=pymake, line_handler=output.on_line,
+                    log=False, print_directory=False)
+
                 # Build target pairs.
                 for make_dir, make_target in target_pairs:
                     # We don't display build status messages during partial
                     # tree builds because they aren't reliable there. This
                     # could potentially be fixed if the build monitor were more
                     # intelligent about encountering undefined state.
                     status = self._run_make(directory=make_dir, target=make_target,
                         line_handler=output.on_line, log=False, print_directory=False,
@@ -488,16 +498,25 @@ class Build(MachCommandBase):
                 if isinstance(e, WindowsError) and e.winerror in (5,32):
                     self.log(logging.ERROR, 'file_access_error', {'error': e},
                         "Could not clobber because a file was in use. If the "
                         "application is running, try closing it. {error}")
                     return 1
 
             raise
 
+    @Command('build-backend', category='build',
+        description='Generate a backend used to build the tree.')
+    def build_backend(self):
+        # When we support multiple build backends (Tup, Visual Studio, etc),
+        # this command will be expanded to support choosing what to generate.
+        config_status = os.path.join(self.topobjdir, 'config.status')
+        return self._run_command_in_objdir(args=[config_status],
+            pass_thru=True, ensure_exit_code=False)
+
 
 @CommandProvider
 class Warnings(MachCommandBase):
     """Provide commands for inspecting warnings."""
 
     @property
     def database_path(self):
         return self._get_state_filename('warnings.json')