Bug 1239217 - Make the RecursiveMake build system create backend files generically. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 22 Jan 2016 10:02:48 +0900
changeset 303637 e9996f9ea66f064494ac7f91aacefd142cb6babc
parent 303636 848ccd8fd7da3ee17089b43e6bff6f24a1f9238c
child 303638 0d610a47138ddf28a34c1cd078f1bfe553af8c96
push id5703
push userraliiev@mozilla.com
push dateMon, 07 Mar 2016 14:18:41 +0000
treeherdermozilla-esr52@31e373ad5b5f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1239217
milestone46.0a1
Bug 1239217 - Make the RecursiveMake build system create backend files generically. r=gps The current rule is only for "backend.RecursiveMakeBackend", but, with the current default of generating both the RecursiveMake and FasterMake backends, the command creates/refreshes both backends. This is, in fact, how the FasterMake backend is refreshed in most cases. Moreover, with an hybrid backends, the generated file is not "backend.RecursiveMakeBackend" anymore, so we need a more generic way to handle this. Furthermore, it's not necessarily desirable for all backends to have a dependency file to handle the dependencies to refresh the backend, so generate a plain list instead. This has the side effect of making `mach build-backend --diff` more readable for changes to that file. Finally, make the backend.* files created like any other backend file, such that its diff appears in the `mach build-backend --diff` output.
Makefile.in
config/rules.mk
python/mozbuild/mozbuild/backend/base.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/mach_commands.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -29,21 +29,23 @@ DIST_GARBAGE = config.cache config.log c
    .mozconfig.mk
 
 ifdef JS_STANDALONE
 configure_dir = $(topsrcdir)/js/src
 else
 configure_dir = $(topsrcdir)
 endif
 
+BUILD_BACKEND_FILES := $(addprefix backend.,$(addsuffix Backend,$(BUILD_BACKENDS)))
+
 ifndef TEST_MOZBUILD
 ifndef MOZ_PROFILE_USE
-# We need to explicitly put backend.RecursiveMakeBackend here
-# otherwise the rule in rules.mk doesn't run early enough.
-$(TIERS) binaries:: CLOBBER $(configure_dir)/configure config.status backend.RecursiveMakeBackend
+# We need to explicitly put BUILD_BACKEND_FILES here otherwise the rule in
+# rules.mk doesn't run early enough.
+$(TIERS) binaries:: CLOBBER $(configure_dir)/configure config.status $(BUILD_BACKEND_FILES)
 ifndef JS_STANDALONE
 ifdef COMPILE_ENVIRONMENT
 $(TIERS) binaries:: $(topsrcdir)/js/src/configure js/src/config.status
 endif
 endif
 endif
 endif
 
@@ -79,37 +81,61 @@ config.status js/src/config.status:
 # Regenerate the build backend if it is out of date. We only have this rule in
 # this main make file because having it in rules.mk and applied to partial tree
 # builds resulted in a world of hurt. Gory details are in bug 877308.
 #
 # The mach build driver will ensure the backend is up to date for partial tree
 # builds. This cleanly avoids most of the pain.
 
 ifndef TEST_MOZBUILD
-backend.RecursiveMakeBackend:
+
+.PHONY: backend
+backend: $(BUILD_BACKEND_FILES)
+
+# A traditional rule would look like this:
+#    backend.%:
+#        @echo do stuff
+#
+# But with -j<n>, and multiple items in BUILD_BACKEND_FILES, the command would
+# run multiple times in parallel.
+#
+# "Fortunately", make has some weird semantics for pattern rules: if there are
+# multiple targets in a pattern rule and each of them is matched at most once,
+# the command will only run once. So:
+#     backend%RecursiveMakeBackend backend%FasterMakeBackend:
+#         @echo do stuff
+#     backend: backend.RecursiveMakeBackend backend.FasterMakeBackend
+# would only execute the command once.
+#
+# Credit where due: http://stackoverflow.com/questions/2973445/gnu-makefile-rule-generating-a-few-targets-from-a-single-source-file/3077254#3077254
+$(subst .,%,$(BUILD_BACKEND_FILES)):
 	@echo 'Build configuration changed. Regenerating backend.'
 	$(PYTHON) config.status
 
-Makefile: backend.RecursiveMakeBackend
+Makefile: $(BUILD_BACKEND_FILES)
 	@$(TOUCH) $@
 
-include backend.RecursiveMakeBackend.pp
+define build_backend_rule
+$(1): $$(shell cat $(1).in)
 
-default:: backend.RecursiveMakeBackend
+endef
+$(foreach file,$(BUILD_BACKEND_FILES),$(eval $(call build_backend_rule,$(file))))
+
+default:: $(BUILD_BACKEND_FILES)
 endif
 
 install_manifests := \
   $(addprefix dist/,bin branding idl include public private sdk xpi-stage) \
   _tests \
   $(NULL)
 install_manifest_depends = \
   CLOBBER \
   $(configure_dir)/configure \
   config.status \
-  backend.RecursiveMakeBackend \
+  $(BUILD_BACKEND_FILES) \
   $(NULL)
 
 ifndef JS_STANDALONE
 ifdef COMPILE_ENVIRONMENT
 install_manifest_depends += \
   $(topsrcdir)/js/src/configure \
   js/src/config.status \
   $(NULL)
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -527,22 +527,27 @@ endif
 ################################################################################
 
 # Ensure the build config is up to date. This is done automatically when builds
 # are performed through |mach build|. The check here is to catch people not
 # using mach. If we ever enforce builds through mach, this code can be removed.
 ifndef MOZBUILD_BACKEND_CHECKED
 ifndef MACH
 ifndef TOPLEVEL_BUILD
-$(DEPTH)/backend.RecursiveMakeBackend:
+BUILD_BACKEND_FILES := $(addprefix $(DEPTH)/backend.,$(addsuffix Backend,$(BUILD_BACKENDS)))
+$(DEPTH)/backend.%Backend:
 	$(error Build configuration changed. Build with |mach build| or run |mach build-backend| to regenerate build config)
 
-include $(DEPTH)/backend.RecursiveMakeBackend.pp
+define build_backend_rule
+$(1): $$(shell cat $(1).in)
 
-default:: $(DEPTH)/backend.RecursiveMakeBackend
+endef
+$(foreach file,$(BUILD_BACKEND_FILES),$(eval $(call build_backend_rule,$(file))))
+
+default:: $(BUILD_BACKEND_FILES)
 
 export MOZBUILD_BACKEND_CHECKED=1
 endif
 endif
 endif
 
 # The root makefile doesn't want to do a plain export/libs, because
 # of the tiers and because of libxul. Suppress the default rules in favor
--- a/python/mozbuild/mozbuild/backend/base.py
+++ b/python/mozbuild/mozbuild/backend/base.py
@@ -49,24 +49,16 @@ class BuildBackend(LoggingMixin):
 
         # Files whose modification should cause a new read and backend
         # generation.
         self.backend_input_files = set()
 
         # Files generated by the backend.
         self._backend_output_files = set()
 
-        # Previously generated files.
-        self._backend_output_list_file = mozpath.join(environment.topobjdir,
-            'backend.%s' % self.__class__.__name__)
-        self._backend_output_list = set()
-        if os.path.exists(self._backend_output_list_file):
-            l = open(self._backend_output_list_file).read().split('\n')
-            self._backend_output_list.update(mozpath.normsep(p) for p in l)
-
         self._environments = {}
         self._environments[environment.topobjdir] = environment
 
         # The number of backend files created.
         self._created_count = 0
 
         # The number of backend files updated.
         self._updated_count = 0
@@ -116,16 +108,26 @@ class BuildBackend(LoggingMixin):
 
         This is the main method of the interface. This is what takes the
         frontend output and does something with it.
 
         Child classes are not expected to implement this method. Instead, the
         base class consumes objects and calls methods (possibly) implemented by
         child classes.
         """
+
+        # Previously generated files.
+        list_file = mozpath.join(self.environment.topobjdir, 'backend.%s'
+                                 % self.__class__.__name__)
+        backend_output_list = set()
+        if os.path.exists(list_file):
+            with open(list_file) as fh:
+                backend_output_list.update(mozpath.normsep(p)
+                                           for p in fh.read().splitlines())
+
         for obj in objs:
             obj_start = time.time()
             if (not self.consume_object(obj) and
                     not isinstance(self, PartialBackend)):
                 raise Exception('Unhandled object of type %s' % type(obj))
             self._execution_time += time.time() - obj_start
 
             if (isinstance(obj, ContextDerived) and
@@ -137,17 +139,17 @@ class BuildBackend(LoggingMixin):
         self.backend_input_files |= set(iter_modules_in_path(
             self.environment.topsrcdir, self.environment.topobjdir))
 
         finished_start = time.time()
         self.consume_finished()
         self._execution_time += time.time() - finished_start
 
         # Purge backend files created in previous run, but not created anymore
-        delete_files = self._backend_output_list - self._backend_output_files
+        delete_files = backend_output_list - self._backend_output_files
         for path in delete_files:
             full_path = mozpath.join(self.environment.topobjdir, path)
             try:
                 with open(full_path, 'r') as existing:
                     old_content = existing.read()
                     if old_content:
                         self.file_diffs[full_path] = simple_diff(
                             full_path, old_content.splitlines(), None)
@@ -163,23 +165,28 @@ class BuildBackend(LoggingMixin):
         for dir in set(mozpath.dirname(d) for d in delete_files):
             try:
                 os.removedirs(dir)
             except OSError:
                 pass
 
         # Write out the list of backend files generated, if it changed.
         if self._deleted_count or self._created_count or \
-                not os.path.exists(self._backend_output_list_file):
-            with open(self._backend_output_list_file, 'w') as fh:
+                not os.path.exists(list_file):
+            with self._write_file(list_file) as fh:
                 fh.write('\n'.join(sorted(self._backend_output_files)))
         else:
             # Always update its mtime.
-            with open(self._backend_output_list_file, 'a'):
-                os.utime(self._backend_output_list_file, None)
+            with open(list_file, 'a'):
+                os.utime(list_file, None)
+
+        # Write out the list of input files for the backend
+        with self._write_file('%s.in' % list_file) as fh:
+            fh.write('\n'.join(sorted(
+                mozpath.normsep(f) for f in self.backend_input_files)))
 
     @abstractmethod
     def consume_object(self, obj):
         """Consumes an individual TreeMetadata instance.
 
         This is the main method used by child classes to react to build
         metadata.
         """
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -819,29 +819,16 @@ class RecursiveMakeBackend(CommonBackend
                             self.environment.topobjdir))
 
                     # Detect any Makefile.ins that contain variables on the
                     # moz.build-only list
                     self._check_blacklisted_variables(makefile_in, content)
 
         self._fill_root_mk()
 
-        # Write out a dependency file used to determine whether a config.status
-        # re-run is needed.
-        inputs = sorted(p.replace(os.sep, '/') for p in self.backend_input_files)
-
-        # We need to use $(DEPTH) so the target here matches what's in
-        # rules.mk. If they are different, the dependencies don't get pulled in
-        # properly.
-        with self._write_file('%s.pp' % self._backend_output_list_file) as backend_deps:
-            backend_deps.write('$(DEPTH)/backend.%s: %s\n' %
-                (self.__class__.__name__, ' '.join(inputs)))
-            for path in inputs:
-                backend_deps.write('%s:\n' % path)
-
         # Make the master test manifest files.
         for flavor, t in self._test_manifests.items():
             install_prefix, manifests = t
             manifest_stem = mozpath.join(install_prefix, '%s.ini' % flavor)
             self._write_master_test_manifest(mozpath.join(
                 self.environment.topobjdir, '_tests', manifest_stem),
                 manifests)
 
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -381,18 +381,17 @@ class Build(MachCommandBase):
                              'instead of {target_pairs}.')
                     target_pairs = new_pairs
 
                 # Ensure build backend is up to date. The alternative is to
                 # have rules in the invoked Makefile to rebuild the build
                 # backend. But that involves make reinvoking itself and there
                 # are undesired side-effects of this. See bug 877308 for a
                 # comprehensive history lesson.
-                self._run_make(directory=self.topobjdir,
-                    target='backend.RecursiveMakeBackend',
+                self._run_make(directory=self.topobjdir, target='backend',
                     line_handler=output.on_line, log=False,
                     print_directory=False)
 
                 if self.substs.get('MOZ_ARTIFACT_BUILDS', False):
                     self._run_mach_artifact_install()
 
                 # Build target pairs.
                 for make_dir, make_target in target_pairs:
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -159,17 +159,17 @@ class TestRecursiveMakeTraversal(unittes
 
 class TestRecursiveMakeBackend(BackendTester):
     def test_basic(self):
         """Ensure the RecursiveMakeBackend works without error."""
         env = self._consume('stub0', RecursiveMakeBackend)
         self.assertTrue(os.path.exists(mozpath.join(env.topobjdir,
             'backend.RecursiveMakeBackend')))
         self.assertTrue(os.path.exists(mozpath.join(env.topobjdir,
-            'backend.RecursiveMakeBackend.pp')))
+            'backend.RecursiveMakeBackend.in')))
 
     def test_output_files(self):
         """Ensure proper files are generated."""
         env = self._consume('stub0', RecursiveMakeBackend)
 
         expected = ['', 'dir1', 'dir2']
 
         for d in expected: