Bug 1498031 - Merge code paths for running configure between Tup and Make based backends. r=firefox-build-system-reviewers,mshal
authorChris Manchester <cmanchester@mozilla.com>
Tue, 16 Oct 2018 22:21:36 +0000
changeset 497275 6dad1580bc3ea76a5239ef86a78e82745f197f5a
parent 497274 e85db490e17a596a40eb8bc2a3d2787a77c41d50
child 497276 55bb23a27ec7e39516d9d7f4254b312220f2ea34
push id9996
push userarchaeopteryx@coole-files.de
push dateThu, 18 Oct 2018 18:37:15 +0000
treeherdermozilla-beta@8efe26839243 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfirefox-build-system-reviewers, mshal
bugs1498031
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1498031 - Merge code paths for running configure between Tup and Make based backends. r=firefox-build-system-reviewers,mshal This addresses a related issue along the way: a build that results in running configure would not update the value of self.config_environment (and therefore self.substs) as seen from the build driver, so out of date values would have been used. The changes to Makefile.in and client.mk made exploit the assumption that by he time anything in the Make build is running, config.status is up to date. Users running "make" without the benefit of "mach" will need to manually run configure when necessary in order to take this into account. Differential Revision: https://phabricator.services.mozilla.com/D8473
Makefile.in
client.mk
python/mozbuild/mozbuild/base.py
python/mozbuild/mozbuild/controller/building.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -40,56 +40,36 @@ endif
 # Additionally, provide a dummy target during tests, because
 # faster/rules.mk will expect these targets to exist.
 ifdef TEST_MOZBUILD
 source-repo.h: $(MDDEPDIR)/source-repo.h.stub
 buildid.h: $(MDDEPDIR)/buildid.h.stub
 endif
 endif
 
-ifdef JS_STANDALONE
-configure_dir = $(topsrcdir)/js/src
-else
-configure_dir = $(topsrcdir)
-endif
-
 BUILD_BACKEND_FILES := $(addprefix backend.,$(addsuffix Backend,$(BUILD_BACKENDS)))
 
 ifndef TEST_MOZBUILD
 ifndef MOZ_PROFILE_USE
 # We need to explicitly put BUILD_BACKEND_FILES here otherwise the rule in
 # rules.mk doesn't run early enough.
-$(TIERS) binaries:: CLOBBER $(configure_dir)/configure config.status $(BUILD_BACKEND_FILES)
-ifdef MOZ_WIDGET_TOOLKIT
-ifdef COMPILE_ENVIRONMENT
-$(TIERS) binaries:: $(topsrcdir)/js/src/configure js/src/config.status
-endif
-endif
+$(TIERS) binaries:: CLOBBER $(BUILD_BACKEND_FILES)
 endif
 endif
 
 ifdef JS_STANDALONE
 .PHONY: CLOBBER
 CLOBBER:
 else
 CLOBBER: $(topsrcdir)/CLOBBER
 	@echo 'STOP!  The CLOBBER file has changed.'
 	@echo 'Please run the build through "mach build".'
 	@exit 1
 endif
 
-config.status: $(configure_dir)/configure $(configure_dir)/old-configure
-js/src/config.status: $(topsrcdir)/js/src/configure $(topsrcdir)/js/src/old-configure
-config.status js/src/config.status:
-	@echo 'STOP!  $? has changed and needs to be run again.'
-	@echo 'Please rerun it.'
-	@echo 'To ignore this message, touch "$(CURDIR)/$@",'
-	@echo 'but your build might not succeed.'
-	@exit 1
-
 # Regenerate the build backend if it is out of date. We only have this rule in
 # this main make file because having it in rules.mk and applied to partial tree
 # builds resulted in a world of hurt. Gory details are in bug 877308.
 #
 # The mach build driver will ensure the backend is up to date for partial tree
 # builds. This cleanly avoids most of the pain.
 
 ifndef TEST_MOZBUILD
@@ -113,30 +93,19 @@ install_manifests := \
 # FasterMake/RecursiveMake backend. This is a hack until bug 1241744 moves
 # xpidl handling to FasterMake in that case, mechanically making the dist/bin
 # install manifest non-existent (non-existent manifests being skipped)
 ifeq (,$(filter FasterMake+RecursiveMake,$(BUILD_BACKENDS)))
 install_manifests += dist/bin
 endif
 install_manifest_depends = \
   CLOBBER \
-  $(configure_dir)/configure \
-  config.status \
   $(BUILD_BACKEND_FILES) \
   $(NULL)
 
-ifdef MOZ_WIDGET_TOOLKIT
-ifdef COMPILE_ENVIRONMENT
-install_manifest_depends += \
-  $(topsrcdir)/js/src/configure \
-  js/src/config.status \
-  $(NULL)
-endif
-endif
-
 .PHONY: install-manifests
 install-manifests: $(addprefix install-,$(install_manifests))
 
 # If we're using the hybrid FasterMake/RecursiveMake backend, we want
 # to recurse in the faster/ directory in parallel of install manifests.
 ifneq (,$(filter FasterMake+RecursiveMake,$(BUILD_BACKENDS)))
 install-manifests: faster
 .PHONY: faster
--- a/client.mk
+++ b/client.mk
@@ -84,22 +84,16 @@ build::
 	# the command doesn't run in that case)
 	mkdir -p $(UPLOAD_PATH)
 	$(if $(findstring n,$(filter-out --%, $(MAKEFLAGS))),,+)env RUST_LOG=sccache=debug SCCACHE_ERROR_LOG=$(UPLOAD_PATH)/sccache.log $(MOZBUILD_MANAGE_SCCACHE_DAEMON) --start-server
 endif
 
 ####################################
 # Configure
 
-MAKEFILE      = $(wildcard $(OBJDIR)/Makefile)
-CONFIG_STATUS = $(wildcard $(OBJDIR)/config.status)
-
-# Include deps for configure written by configure itself.
-CONFIG_STATUS_DEPS := $(if $(wildcard $(OBJDIR)/config_status_deps.in),$(shell cat $(OBJDIR)/config_status_deps.in),)
-
 $(CONFIGURES): %: %.in
 	@echo Generating $@
 	cp -f $< $@
 	chmod +x $@
 
 CONFIGURE_ENV_ARGS += \
   MAKE='$(MAKE)' \
   $(NULL)
@@ -108,51 +102,31 @@ CONFIGURE_ENV_ARGS += \
 #   $(TOPSRCDIR) will set @srcdir@ to "."; otherwise, it is set to the full
 #   path of $(TOPSRCDIR).
 ifeq ($(TOPSRCDIR),$(OBJDIR))
   CONFIGURE = ./configure
 else
   CONFIGURE = $(TOPSRCDIR)/configure
 endif
 
-configure-files: $(CONFIGURES)
-
-configure-preqs = \
-  configure-files \
-  $(OBJDIR)/.mozconfig.json \
-  $(NULL)
-
-configure:: $(configure-preqs)
+configure:: $(CONFIGURES)
 	$(call BUILDSTATUS,TIERS configure)
 	$(call BUILDSTATUS,TIER_START configure)
 	@echo cd $(OBJDIR);
 	@echo $(CONFIGURE) $(CONFIGURE_ARGS)
 	@cd $(OBJDIR) && $(CONFIGURE_ENV_ARGS) $(CONFIGURE) $(CONFIGURE_ARGS) \
 	  || ( echo '*** Fix above errors and then restart with\
                "./mach build"' && exit 1 )
 	@touch $(OBJDIR)/Makefile
 	$(call BUILDSTATUS,TIER_FINISH configure)
 
-ifneq (,$(MAKEFILE))
-$(OBJDIR)/Makefile: $(OBJDIR)/config.status
-
-
-
-
-
-$(OBJDIR)/config.status: $(CONFIG_STATUS_DEPS)
-else
-$(OBJDIR)/Makefile: $(CONFIG_STATUS_DEPS)
-endif
-	@$(MAKE) -f $(TOPSRCDIR)/client.mk configure
-
 ####################################
 # Build it
 
-build::  $(OBJDIR)/Makefile $(OBJDIR)/config.status
+build::
 	+$(MOZ_MAKE)
 
 ifdef MOZ_AUTOMATION
 build::
 	+$(MOZ_MAKE) automation/build
 endif
 
 # This makefile doesn't support parallel execution. It does pass
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -312,16 +312,23 @@ class MozbuildObject(ProcessExecutionMix
         # If we don't have a configure context, fall back to auto-detection.
         try:
             return get_repository_from_build_config(self)
         except BuildEnvironmentNotFoundException:
             pass
 
         return get_repository_object(self.topsrcdir)
 
+    def reload_config_environment(self):
+        '''Force config.status to be re-read and return the new value
+        of ``self.config_environment``.
+        '''
+        self._config_environment = None
+        return self.config_environment
+
     def mozbuild_reader(self, config_mode='build', vcs_revision=None,
                         vcs_check_clean=True):
         """Obtain a ``BuildReader`` for evaluating moz.build files.
 
         Given arguments, returns a ``mozbuild.frontend.reader.BuildReader``
         that can be used to evaluate moz.build files for this repo.
 
         ``config_mode`` is either ``build`` or ``empty``. If ``build``,
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -1034,47 +1034,55 @@ class BuildDriver(MozbuildObject):
             monitor.start_resource_recording()
 
             config = None
             try:
                 config = self.config_environment
             except Exception:
                 pass
 
-            if config is None:
+            # Record whether a clobber was requested so we can print
+            # a special message later if the build fails.
+            clobber_requested = False
+
+            # Write out any changes to the current mozconfig in case
+            # they should invalidate configure.
+            self._write_mozconfig_json()
+
+            previous_backend = None
+            if config is not None:
+                previous_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
+
+            config_rc = None
+            # Even if we have a config object, it may be out of date
+            # if something that influences its result has changed.
+            if config is None or build_out_of_date(mozpath.join(self.topobjdir,
+                                                                'config.status'),
+                                                   mozpath.join(self.topobjdir,
+                                                                'config_status_deps.in')):
+                if previous_backend and 'Make' not in previous_backend:
+                    clobber_requested = self._clobber_configure()
+
                 config_rc = self.configure(buildstatus_messages=True,
                                            line_handler=output.on_line)
+
                 if config_rc != 0:
                     return config_rc
 
-                config = self.config_environment
+                config = self.reload_config_environment()
+
+            active_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
 
             status = None
-            active_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
-            if active_backend and 'Make' not in active_backend:
-                # Record whether a clobber was requested so we can print
-                # a special message later if the build fails.
-                clobber_requested = False
-                # Write out any changes to the current mozconfig in case
-                # they should invalidate configure.
-                self._write_mozconfig_json()
-                # Even if we have a config object, it may be out of date
-                # if something that influences its result has changed.
-                if build_out_of_date(mozpath.join(self.topobjdir,
-                                                  'config.status'),
-                                     mozpath.join(self.topobjdir,
-                                                  'config_status_deps.in')):
-                    clobber_requested = self._clobber_configure()
-                    config_rc = self.configure(buildstatus_messages=True,
-                                               line_handler=output.on_line)
-                    if config_rc != 0:
-                        return config_rc
-                elif backend_out_of_date(mozpath.join(self.topobjdir,
-                                                      'backend.%sBackend' %
-                                                      active_backend)):
+
+            if 'Make' not in active_backend:
+                if (not config_rc and
+                    backend_out_of_date(mozpath.join(self.topobjdir,
+                                                     'backend.%sBackend' %
+                                                     active_backend))):
                     print('Build configuration changed. Regenerating backend.')
                     args = [config.substs['PYTHON'],
                             mozpath.join(self.topobjdir, 'config.status')]
                     self.run_process(args, cwd=self.topobjdir, pass_thru=True)
 
                 # client.mk has its own handling of MOZ_PARALLEL_BUILD so the
                 # make backend can determine when to run in single-threaded mode
                 # or parallel mode. For other backends, we can pass in the value
@@ -1174,17 +1182,16 @@ class BuildDriver(MozbuildObject):
                                              keep_going=keep_going)
 
             self.log(logging.WARNING, 'warning_summary',
                      {'count': len(monitor.warnings_database)},
                      '{count} compiler warnings present.')
 
             # Try to run the active build backend's post-build step, if possible.
             try:
-                config = self.config_environment
                 active_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
                 if active_backend:
                     backend_cls = get_backend_class(active_backend)(config)
                     new_status = backend_cls.post_build(self, output, jobs, verbose, status)
                     status = new_status
             except Exception as ex:
                 self.log(logging.DEBUG, 'post_build', {'ex': str(ex)},
                          "Unable to run active build backend's post-build step; " +
@@ -1451,18 +1458,16 @@ class BuildDriver(MozbuildObject):
                                            '.mozconfig-client-mk')
         with FileAvoidWrite(mozconfig_client_mk) as fh:
             fh.write(b'\n'.join(mozconfig_make_lines))
 
         mozconfig_mk = os.path.join(self.topobjdir, '.mozconfig.mk')
         with FileAvoidWrite(mozconfig_mk) as fh:
             fh.write(b'\n'.join(mozconfig_filtered_lines))
 
-        self._write_mozconfig_json()
-
         # Copy the original mozconfig to the objdir.
         mozconfig_objdir = os.path.join(self.topobjdir, '.mozconfig')
         if mozconfig['path']:
             with open(mozconfig['path'], 'rb') as ifh:
                 with FileAvoidWrite(mozconfig_objdir) as ofh:
                     ofh.write(ifh.read())
         else:
             try: