Bug 1537574 - Use mozbuild's backend-out-of-date logic for RecursiveMake; r=firefox-build-system-reviewers,chmanchester
☠☠ backed out by facb0f655f03 ☠ ☠
authorMike Shal <marfey@gmail.com>
Mon, 08 Apr 2019 16:37:56 +0000
changeset 530268 95b3298fd2d462ecf0aab134289838317009fb3c
parent 530267 1c0a3a443f9734d543535c61bdc3ceb1ad3002b3
child 530269 bba4f161674c8e119b45e8a2f4b6b758c229f8b5
push id2082
push userffxbld-merge
push dateMon, 01 Jul 2019 08:34:18 +0000
treeherdermozilla-release@2fb19d0466d2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfirefox-build-system-reviewers, chmanchester
bugs1537574
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1537574 - Use mozbuild's backend-out-of-date logic for RecursiveMake; r=firefox-build-system-reviewers,chmanchester If mozbuild parsing fails due to a missing file (eg: a file not existing in UNIFIED_SOURCES), then no Makefiles are written out, but config.status exists. This would cause mozbuild to think that configure doesn't need to run, and rely on make to perform the backend-out-of-date check in rebuild-backend.mk. Unfortunately since no Makefiles were written, the make command fails immediately and no attempt is made to re-create the backend. Note that this is only a problem if the first mozbuild parsing from a clobber build fails, otherwise there is typically a top-level Makefile from a previous build to call into (at which point make can determine it is out-of-date, and re-invoke itself). The fix is to have the RecursiveMake backend re-use the same logic that was introduced into mozbuild for alternate backends, and remove rebuild-backend.mk. This way, mozbuild can always determine if the backend needs to be regenerated, even if the initial parsing failed. Test code was also relying on rebuild-backend.mk to generate the TestBackend, but moving backend_out_of_date() into MozbuildObject allows this code to be shared. Differential Revision: https://phabricator.services.mozilla.com/D26262
Makefile.in
build/gen_test_backend.py
build/rebuild-backend.mk
python/mozbuild/mozbuild/base.py
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/gen_test_backend.py
testing/mozbase/moztest/moztest/resolve.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -60,36 +60,16 @@ ifdef JS_STANDALONE
 CLOBBER:
 else
 CLOBBER: $(topsrcdir)/CLOBBER
 	@echo 'STOP!  The CLOBBER file has changed.'
 	@echo 'Please run the build through "mach build".'
 	@exit 1
 endif
 
-# Regenerate the build backend if it is out of date. We only have this rule in
-# this main make file because having it in rules.mk and applied to partial tree
-# builds resulted in a world of hurt. Gory details are in bug 877308.
-#
-# The mach build driver will ensure the backend is up to date for partial tree
-# builds. This cleanly avoids most of the pain.
-
-ifndef TEST_MOZBUILD
-
-.PHONY: backend
-backend: $(BUILD_BACKEND_FILES)
-
-include $(topsrcdir)/build/rebuild-backend.mk
-
-Makefile: $(BUILD_BACKEND_FILES)
-	@$(TOUCH) $@
-
-default:: $(BUILD_BACKEND_FILES)
-endif
-
 install_manifests := \
   $(addprefix dist/,branding include public private xpi-stage) \
   _tests \
   $(NULL)
 # Skip the dist/bin install manifest when using the hybrid
 # FasterMake/RecursiveMake backend. This is a hack until bug 1241744 moves
 # xpidl handling to FasterMake in that case, mechanically making the dist/bin
 # install manifest non-existent (non-existent manifests being skipped)
deleted file mode 100644
--- a/build/rebuild-backend.mk
+++ /dev/null
@@ -1,31 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-BACKEND_GENERATION_SCRIPT ?= config.status
-
-# A traditional rule would look like this:
-#    backend.%:
-#        @echo do stuff
-#
-# But with -j<n>, and multiple items in BUILD_BACKEND_FILES, the command would
-# run multiple times in parallel.
-#
-# "Fortunately", make has some weird semantics for pattern rules: if there are
-# multiple targets in a pattern rule and each of them is matched at most once,
-# the command will only run once. So:
-#     backend%RecursiveMakeBackend backend%FasterMakeBackend:
-#         @echo do stuff
-#     backend: backend.RecursiveMakeBackend backend.FasterMakeBackend
-# would only execute the command once.
-#
-# Credit where due: http://stackoverflow.com/questions/2973445/gnu-makefile-rule-generating-a-few-targets-from-a-single-source-file/3077254#3077254
-$(subst .,%,$(BUILD_BACKEND_FILES)):
-	@echo 'Build configuration changed. Regenerating backend.'
-	$(PYTHON) $(BACKEND_GENERATION_SCRIPT)
-
-define build_backend_rule
-$(1): $$(wildcard $$(shell cat $(1).in))
-
-endef
-$(foreach file,$(BUILD_BACKEND_FILES),$(eval $(call build_backend_rule,$(file))))
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -192,16 +192,59 @@ class MozbuildObject(ProcessExecutionMix
             topobjdir = topobjdir.replace('@CONFIG_GUESS@',
                 self.resolve_config_guess())
 
         if not os.path.isabs(topobjdir):
             topobjdir = os.path.abspath(os.path.join(self.topsrcdir, topobjdir))
 
         return mozpath.normsep(os.path.normpath(topobjdir))
 
+    def build_out_of_date(self, output, dep_file):
+        if not os.path.isfile(output):
+            print(" Output reference file not found: %s" % output)
+            return True
+        if not os.path.isfile(dep_file):
+            print(" Dependency file not found: %s" % dep_file)
+            return True
+
+        deps = []
+        with open(dep_file, 'r') as fh:
+            deps = fh.read().splitlines()
+
+        mtime = os.path.getmtime(output)
+        for f in deps:
+            try:
+                dep_mtime = os.path.getmtime(f)
+            except OSError as e:
+                if e.errno == errno.ENOENT:
+                    print(" Input not found: %s" % f)
+                    return True
+                raise
+            if dep_mtime > mtime:
+                print(" %s is out of date with respect to %s" % (output, f))
+                return True
+        return False
+
+    def backend_out_of_date(self, backend_file):
+        if not os.path.isfile(backend_file):
+            return True
+
+        # Check if any of our output files have been removed since
+        # we last built the backend, re-generate the backend if
+        # so.
+        outputs = []
+        with open(backend_file, 'r') as fh:
+            outputs = fh.read().splitlines()
+        for output in outputs:
+            if not os.path.isfile(mozpath.join(self.topobjdir, output)):
+                return True
+
+        dep_file = '%s.in' % backend_file
+        return self.build_out_of_date(backend_file, dep_file)
+
     @property
     def topobjdir(self):
         if self._topobjdir is None:
             self._topobjdir = self.resolve_mozconfig_topobjdir(
                 default='obj-@CONFIG_GUESS@')
 
         return self._topobjdir
 
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -1009,59 +1009,16 @@ class BuildDriver(MozbuildObject):
                 return 1
 
             if directory is not None:
                 disable_extra_make_dependencies=True
                 directory = mozpath.normsep(directory)
                 if directory.startswith('/'):
                     directory = directory[1:]
 
-            def build_out_of_date(output, dep_file):
-                if not os.path.isfile(output):
-                    print(" Output reference file not found: %s" % output)
-                    return True
-                if not os.path.isfile(dep_file):
-                    print(" Configure dependency file not found: %s" % dep_file)
-                    return True
-
-                deps = []
-                with open(dep_file, 'r') as fh:
-                    deps = fh.read().splitlines()
-
-                mtime = os.path.getmtime(output)
-                for f in deps:
-                    try:
-                        dep_mtime = os.path.getmtime(f)
-                    except OSError as e:
-                        if e.errno == errno.ENOENT:
-                            print(" Configure input not found: %s" % f)
-                            return True
-                        raise
-                    if dep_mtime > mtime:
-                        print(" %s is out of date with respect to %s" % (output, f))
-                        return True
-                return False
-
-            def backend_out_of_date(backend_file):
-                if not os.path.isfile(backend_file):
-                    return True
-
-                # Check if any of our output files have been removed since
-                # we last built the backend, re-generate the backend if
-                # so.
-                outputs = []
-                with open(backend_file, 'r') as fh:
-                    outputs = fh.read().splitlines()
-                for output in outputs:
-                    if not os.path.isfile(mozpath.join(self.topobjdir, output)):
-                        return True
-
-                dep_file = '%s.in' % backend_file
-                return build_out_of_date(backend_file, dep_file)
-
             monitor.start_resource_recording()
 
             self.mach_context.command_attrs['clobber'] = False
             config = None
             try:
                 config = self.config_environment
             except Exception:
                 # If we don't already have a config environment this is either
@@ -1079,20 +1036,20 @@ class BuildDriver(MozbuildObject):
 
             previous_backend = None
             if config is not None:
                 previous_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
 
             config_rc = None
             # Even if we have a config object, it may be out of date
             # if something that influences its result has changed.
-            if config is None or build_out_of_date(mozpath.join(self.topobjdir,
-                                                                'config.status'),
-                                                   mozpath.join(self.topobjdir,
-                                                                'config_status_deps.in')):
+            if config is None or self.build_out_of_date(mozpath.join(self.topobjdir,
+                                                                     'config.status'),
+                                                        mozpath.join(self.topobjdir,
+                                                                     'config_status_deps.in')):
                 if previous_backend and 'Make' not in previous_backend:
                     clobber_requested = self._clobber_configure()
 
                 if config is None:
                     print(" Config object not found by mach.")
 
                 config_rc = self.configure(buildstatus_messages=True,
                                            line_handler=output.on_line)
@@ -1101,26 +1058,26 @@ class BuildDriver(MozbuildObject):
                     return config_rc
 
                 config = self.reload_config_environment()
 
             active_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
 
             status = None
 
+            if (not config_rc and
+                self.backend_out_of_date(mozpath.join(self.topobjdir,
+                                                      'backend.%sBackend' %
+                                                      active_backend))):
+                print('Build configuration changed. Regenerating backend.')
+                args = [config.substs['PYTHON'],
+                        mozpath.join(self.topobjdir, 'config.status')]
+                self.run_process(args, cwd=self.topobjdir, pass_thru=True)
+
             if 'Make' not in active_backend:
-                if (not config_rc and
-                    backend_out_of_date(mozpath.join(self.topobjdir,
-                                                     'backend.%sBackend' %
-                                                     active_backend))):
-                    print('Build configuration changed. Regenerating backend.')
-                    args = [config.substs['PYTHON'],
-                            mozpath.join(self.topobjdir, 'config.status')]
-                    self.run_process(args, cwd=self.topobjdir, pass_thru=True)
-
                 # client.mk has its own handling of MOZ_PARALLEL_BUILD so the
                 # make backend can determine when to run in single-threaded mode
                 # or parallel mode. For other backends, we can pass in the value
                 # of MOZ_PARALLEL_BUILD if -jX was not specified on the
                 # commandline.
                 if jobs == 0 and 'make_extra' in self.mozconfig:
                     for param in self.mozconfig['make_extra']:
                         key, value = param.split('=')
@@ -1176,25 +1133,16 @@ class BuildDriver(MozbuildObject):
                     new_pairs = list(add_extra_dependencies(target_pairs, dm))
                     self.log(logging.DEBUG, 'dumbmake',
                              {'target_pairs': target_pairs,
                               'new_pairs': new_pairs},
                              'Added extra dependencies: will build {new_pairs} ' +
                              'instead of {target_pairs}.')
                     target_pairs = new_pairs
 
-                # Ensure build backend is up to date. The alternative is to
-                # have rules in the invoked Makefile to rebuild the build
-                # backend. But that involves make reinvoking itself and there
-                # are undesired side-effects of this. See bug 877308 for a
-                # comprehensive history lesson.
-                self._run_make(directory=self.topobjdir, target='backend',
-                    line_handler=output.on_line, log=False,
-                    print_directory=False, keep_going=keep_going)
-
                 # Build target pairs.
                 for make_dir, make_target in target_pairs:
                     # We don't display build status messages during partial
                     # tree builds because they aren't reliable there. This
                     # could potentially be fixed if the build monitor were more
                     # intelligent about encountering undefined state.
                     no_build_status = b'1' if make_dir is not None else b''
                     status = self._run_make(directory=make_dir, target=make_target,
rename from build/gen_test_backend.py
rename to python/mozbuild/mozbuild/gen_test_backend.py
--- a/testing/mozbase/moztest/moztest/resolve.py
+++ b/testing/mozbase/moztest/moztest/resolve.py
@@ -539,26 +539,22 @@ class TestResolver(MozbuildObject):
     """Helper to resolve tests from the current environment to test files."""
 
     def __init__(self, *args, **kwargs):
         MozbuildObject.__init__(self, *args, **kwargs)
 
         # If installing tests is going to result in re-generating the build
         # backend, we need to do this here, so that the updated contents of
         # all-tests.pkl make it to the set of tests to run.
-        self._run_make(
-            target='backend.TestManifestBackend', pass_thru=True, print_directory=False,
-            filename=mozpath.join(self.topsrcdir, 'build', 'rebuild-backend.mk'),
-            append_env={
-                b'PYTHON': self.virtualenv_manager.python_path,
-                b'BUILD_BACKEND_FILES': b'backend.TestManifestBackend',
-                b'BACKEND_GENERATION_SCRIPT': mozpath.join(
-                    self.topsrcdir, 'build', 'gen_test_backend.py'),
-            },
-        )
+        if self.backend_out_of_date(mozpath.join(self.topobjdir,
+                                                 'backend.TestManifestBackend'
+                                                 )):
+            print("Test configuration changed. Regenerating backend.")
+            from mozbuild.gen_test_backend import gen_test_backend
+            gen_test_backend()
 
         self._tests = TestMetadata(os.path.join(self.topobjdir,
                                                 'all-tests.pkl'),
                                    self.topsrcdir,
                                    test_defaults=os.path.join(self.topobjdir,
                                                               'test-defaults.pkl'))
 
         self._test_rewrites = {