Bug 1537574 - Use mozbuild's backend-out-of-date logic for RecursiveMake; r?#firefox-build-system-reviewers draft
authorMike Shal <marfey@gmail.com>
Wed, 03 Apr 2019 13:27:55 -0700
changeset 1945978 0d0e73fbd7b01649b596316339b53e9d7d7f6607
parent 1942859 a5a5ddfb5178f8a2b30d186d3cf478da38f324ba
child 1945979 32e53e9c0f491dcc5a25d03507a347e5e3d82f22
push id347445
push usermshal@mozilla.com
push dateWed, 03 Apr 2019 20:40:37 +0000
treeherdertry@32e53e9c0f49 [default view] [failures only]
bugs1537574
milestone68.0a1
Bug 1537574 - Use mozbuild's backend-out-of-date logic for RecursiveMake; r?#firefox-build-system-reviewers If mozbuild parsing fails due to a missing file (eg: a file not existing in UNIFIED_SOURCES), then no Makefiles are written out, but config.status exists. This would cause mozbuild to think that configure doesn't need to run, and rely on make to perform the backend-out-of-date check in rebuild-backend.mk. Unfortunately since no Makefiles were written, the make command fails immediately and no attempt is made to re-create the backend. Note that this is only a problem if the first mozbuild parsing from a clobber build fails, otherwise there is typically a top-level Makefile from a previous build to call into (at which point make can determine it is out-of-date, and re-invoke itself). The fix is to have the RecursiveMake backend re-use the same logic that was introduced into mozbuild for alternate backends, and remove rebuild-backend.mk. This way, mozbuild can always determine if the backend needs to be regenerated, even if the initial parsing failed. Test code was also relying on rebuild-backend.mk to generate the TestBackend, but moving backend_out_of_date() into MozbuildObject allows this code to be shared.
Makefile.in
build/gen_test_backend.py
build/rebuild-backend.mk
python/mozbuild/mozbuild/base.py
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/gen_test_backend.py
testing/mozbase/moztest/moztest/resolve.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -60,36 +60,16 @@ ifdef JS_STANDALONE
 CLOBBER:
 else
 CLOBBER: $(topsrcdir)/CLOBBER
 	@echo 'STOP!  The CLOBBER file has changed.'
 	@echo 'Please run the build through "mach build".'
 	@exit 1
 endif
 
-# Regenerate the build backend if it is out of date. We only have this rule in
-# this main make file because having it in rules.mk and applied to partial tree
-# builds resulted in a world of hurt. Gory details are in bug 877308.
-#
-# The mach build driver will ensure the backend is up to date for partial tree
-# builds. This cleanly avoids most of the pain.
-
-ifndef TEST_MOZBUILD
-
-.PHONY: backend
-backend: $(BUILD_BACKEND_FILES)
-
-include $(topsrcdir)/build/rebuild-backend.mk
-
-Makefile: $(BUILD_BACKEND_FILES)
-	@$(TOUCH) $@
-
-default:: $(BUILD_BACKEND_FILES)
-endif
-
 install_manifests := \
   $(addprefix dist/,branding include public private xpi-stage) \
   _tests \
   $(NULL)
 # Skip the dist/bin install manifest when using the hybrid
 # FasterMake/RecursiveMake backend. This is a hack until bug 1241744 moves
 # xpidl handling to FasterMake in that case, mechanically making the dist/bin
 # install manifest non-existent (non-existent manifests being skipped)
deleted file mode 100644
--- a/build/rebuild-backend.mk
+++ /dev/null
@@ -1,31 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-BACKEND_GENERATION_SCRIPT ?= config.status
-
-# A traditional rule would look like this:
-#    backend.%:
-#        @echo do stuff
-#
-# But with -j<n>, and multiple items in BUILD_BACKEND_FILES, the command would
-# run multiple times in parallel.
-#
-# "Fortunately", make has some weird semantics for pattern rules: if there are
-# multiple targets in a pattern rule and each of them is matched at most once,
-# the command will only run once. So:
-#     backend%RecursiveMakeBackend backend%FasterMakeBackend:
-#         @echo do stuff
-#     backend: backend.RecursiveMakeBackend backend.FasterMakeBackend
-# would only execute the command once.
-#
-# Credit where due: http://stackoverflow.com/questions/2973445/gnu-makefile-rule-generating-a-few-targets-from-a-single-source-file/3077254#3077254
-$(subst .,%,$(BUILD_BACKEND_FILES)):
-	@echo 'Build configuration changed. Regenerating backend.'
-	$(PYTHON) $(BACKEND_GENERATION_SCRIPT)
-
-define build_backend_rule
-$(1): $$(wildcard $$(shell cat $(1).in))
-
-endef
-$(foreach file,$(BUILD_BACKEND_FILES),$(eval $(call build_backend_rule,$(file))))
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -192,16 +192,59 @@ class MozbuildObject(ProcessExecutionMix
             topobjdir = topobjdir.replace('@CONFIG_GUESS@',
                 self.resolve_config_guess())
 
         if not os.path.isabs(topobjdir):
             topobjdir = os.path.abspath(os.path.join(self.topsrcdir, topobjdir))
 
         return mozpath.normsep(os.path.normpath(topobjdir))
 
+    def build_out_of_date(self, output, dep_file):
+        if not os.path.isfile(output):
+            print(" Output reference file not found: %s" % output)
+            return True
+        if not os.path.isfile(dep_file):
+            print(" Configure dependency file not found: %s" % dep_file)
+            return True
+
+        deps = []
+        with open(dep_file, 'r') as fh:
+            deps = fh.read().splitlines()
+
+        mtime = os.path.getmtime(output)
+        for f in deps:
+            try:
+                dep_mtime = os.path.getmtime(f)
+            except OSError as e:
+                if e.errno == errno.ENOENT:
+                    print(" Configure input not found: %s" % f)
+                    return True
+                raise
+            if dep_mtime > mtime:
+                print(" %s is out of date with respect to %s" % (output, f))
+                return True
+        return False
+
+    def backend_out_of_date(self, backend_file):
+        if not os.path.isfile(backend_file):
+            return True
+
+        # Check if any of our output files have been removed since
+        # we last built the backend, re-generate the backend if
+        # so.
+        outputs = []
+        with open(backend_file, 'r') as fh:
+            outputs = fh.read().splitlines()
+        for output in outputs:
+            if not os.path.isfile(mozpath.join(self.topobjdir, output)):
+                return True
+
+        dep_file = '%s.in' % backend_file
+        return self.build_out_of_date(backend_file, dep_file)
+
     @property
     def topobjdir(self):
         if self._topobjdir is None:
             self._topobjdir = self.resolve_mozconfig_topobjdir(
                 default='obj-@CONFIG_GUESS@')
 
         return self._topobjdir
 
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -1009,59 +1009,16 @@ class BuildDriver(MozbuildObject):
                 return 1
 
             if directory is not None:
                 disable_extra_make_dependencies=True
                 directory = mozpath.normsep(directory)
                 if directory.startswith('/'):
                     directory = directory[1:]
 
-            def build_out_of_date(output, dep_file):
-                if not os.path.isfile(output):
-                    print(" Output reference file not found: %s" % output)
-                    return True
-                if not os.path.isfile(dep_file):
-                    print(" Configure dependency file not found: %s" % dep_file)
-                    return True
-
-                deps = []
-                with open(dep_file, 'r') as fh:
-                    deps = fh.read().splitlines()
-
-                mtime = os.path.getmtime(output)
-                for f in deps:
-                    try:
-                        dep_mtime = os.path.getmtime(f)
-                    except OSError as e:
-                        if e.errno == errno.ENOENT:
-                            print(" Configure input not found: %s" % f)
-                            return True
-                        raise
-                    if dep_mtime > mtime:
-                        print(" %s is out of date with respect to %s" % (output, f))
-                        return True
-                return False
-
-            def backend_out_of_date(backend_file):
-                if not os.path.isfile(backend_file):
-                    return True
-
-                # Check if any of our output files have been removed since
-                # we last built the backend, re-generate the backend if
-                # so.
-                outputs = []
-                with open(backend_file, 'r') as fh:
-                    outputs = fh.read().splitlines()
-                for output in outputs:
-                    if not os.path.isfile(mozpath.join(self.topobjdir, output)):
-                        return True
-
-                dep_file = '%s.in' % backend_file
-                return build_out_of_date(backend_file, dep_file)
-
             monitor.start_resource_recording()
 
             self.mach_context.command_attrs['clobber'] = False
             config = None
             try:
                 config = self.config_environment
             except Exception:
                 # If we don't already have a config environment this is either
@@ -1079,20 +1036,20 @@ class BuildDriver(MozbuildObject):
 
             previous_backend = None
             if config is not None:
                 previous_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
 
             config_rc = None
             # Even if we have a config object, it may be out of date
             # if something that influences its result has changed.
-            if config is None or build_out_of_date(mozpath.join(self.topobjdir,
-                                                                'config.status'),
-                                                   mozpath.join(self.topobjdir,
-                                                                'config_status_deps.in')):
+            if config is None or self.build_out_of_date(mozpath.join(self.topobjdir,
+                                                                     'config.status'),
+                                                        mozpath.join(self.topobjdir,
+                                                                     'config_status_deps.in')):
                 if previous_backend and 'Make' not in previous_backend:
                     clobber_requested = self._clobber_configure()
 
                 if config is None:
                     print(" Config object not found by mach.")
 
                 config_rc = self.configure(buildstatus_messages=True,
                                            line_handler=output.on_line)
@@ -1101,26 +1058,26 @@ class BuildDriver(MozbuildObject):
                     return config_rc
 
                 config = self.reload_config_environment()
 
             active_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
 
             status = None
 
+            if (not config_rc and
+                self.backend_out_of_date(mozpath.join(self.topobjdir,
+                                                      'backend.%sBackend' %
+                                                      active_backend))):
+                print('Build configuration changed. Regenerating backend.')
+                args = [config.substs['PYTHON'],
+                        mozpath.join(self.topobjdir, 'config.status')]
+                self.run_process(args, cwd=self.topobjdir, pass_thru=True)
+
             if 'Make' not in active_backend:
-                if (not config_rc and
-                    backend_out_of_date(mozpath.join(self.topobjdir,
-                                                     'backend.%sBackend' %
-                                                     active_backend))):
-                    print('Build configuration changed. Regenerating backend.')
-                    args = [config.substs['PYTHON'],
-                            mozpath.join(self.topobjdir, 'config.status')]
-                    self.run_process(args, cwd=self.topobjdir, pass_thru=True)
-
                 # client.mk has its own handling of MOZ_PARALLEL_BUILD so the
                 # make backend can determine when to run in single-threaded mode
                 # or parallel mode. For other backends, we can pass in the value
                 # of MOZ_PARALLEL_BUILD if -jX was not specified on the
                 # commandline.
                 if jobs == 0 and 'make_extra' in self.mozconfig:
                     for param in self.mozconfig['make_extra']:
                         key, value = param.split('=')
rename from build/gen_test_backend.py
rename to python/mozbuild/mozbuild/gen_test_backend.py
--- a/testing/mozbase/moztest/moztest/resolve.py
+++ b/testing/mozbase/moztest/moztest/resolve.py
@@ -539,26 +539,22 @@ class TestResolver(MozbuildObject):
     """Helper to resolve tests from the current environment to test files."""
 
     def __init__(self, *args, **kwargs):
         MozbuildObject.__init__(self, *args, **kwargs)
 
         # If installing tests is going to result in re-generating the build
         # backend, we need to do this here, so that the updated contents of
         # all-tests.pkl make it to the set of tests to run.
-        self._run_make(
-            target='backend.TestManifestBackend', pass_thru=True, print_directory=False,
-            filename=mozpath.join(self.topsrcdir, 'build', 'rebuild-backend.mk'),
-            append_env={
-                b'PYTHON': self.virtualenv_manager.python_path,
-                b'BUILD_BACKEND_FILES': b'backend.TestManifestBackend',
-                b'BACKEND_GENERATION_SCRIPT': mozpath.join(
-                    self.topsrcdir, 'build', 'gen_test_backend.py'),
-            },
-        )
+        if self.backend_out_of_date(mozpath.join(self.topobjdir,
+                                                 'backend.TestManifestBackend'
+                                                 )):
+            print("Test configuration changed. Regenerating backend.")
+            from mozbuild.gen_test_backend import gen_test_backend
+            gen_test_backend()
 
         self._tests = TestMetadata(os.path.join(self.topobjdir,
                                                 'all-tests.pkl'),
                                    self.topsrcdir,
                                    test_defaults=os.path.join(self.topobjdir,
                                                               'test-defaults.pkl'))
 
         self._test_rewrites = {