Bug 1239217 - Add the FasterMake+RecursiveMake hybrid backend. r=gps
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 22 Jan 2016 11:13:26 +0900
changeset 303638 0d610a47138ddf28a34c1cd078f1bfe553af8c96
parent 303637 e9996f9ea66f064494ac7f91aacefd142cb6babc
child 303639 ef27c15ebe8758a7bc701fdb45a9018e8718710e
push id5703
push userraliiev@mozilla.com
push dateMon, 07 Mar 2016 14:18:41 +0000
treeherdermozilla-esr52@31e373ad5b5f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1239217
milestone46.0a1
Bug 1239217 - Add the FasterMake+RecursiveMake hybrid backend. r=gps
Makefile.in
config/faster/rules.mk
python/mozbuild/mozbuild/backend/__init__.py
python/mozbuild/mozbuild/backend/fastermake.py
python/mozbuild/mozbuild/test/backend/test_build.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -118,19 +118,26 @@ define build_backend_rule
 
 endef
 $(foreach file,$(BUILD_BACKEND_FILES),$(eval $(call build_backend_rule,$(file))))
 
 default:: $(BUILD_BACKEND_FILES)
 endif
 
 install_manifests := \
-  $(addprefix dist/,bin branding idl include public private sdk xpi-stage) \
+  $(addprefix dist/,branding idl include public private sdk xpi-stage) \
   _tests \
   $(NULL)
+# Skip the dist/bin install manifest when using the hybrid
+# FasterMake/RecursiveMake backend. This is a hack until bug 1241744 moves
+# xpidl handling to FasterMake in that case, mechanically making the dist/bin
+# install manifest non-existent (non-existent manifests being skipped)
+ifeq (,$(filter FasterMake+RecursiveMake,$(BUILD_BACKENDS)))
+install_manifests += dist/bin
+endif
 install_manifest_depends = \
   CLOBBER \
   $(configure_dir)/configure \
   config.status \
   $(BUILD_BACKEND_FILES) \
   $(NULL)
 
 ifndef JS_STANDALONE
@@ -140,27 +147,42 @@ install_manifest_depends += \
   js/src/config.status \
   $(NULL)
 endif
 endif
 
 .PHONY: install-manifests
 install-manifests: $(addprefix install-,$(install_manifests))
 
+# If we're using the hybrid FasterMake/RecursiveMake backend, we want
+# to recurse in the faster/ directory in parallel of install manifests.
+ifneq (,$(filter FasterMake+RecursiveMake,$(BUILD_BACKENDS)))
+install-manifests: faster
+.PHONY: faster
+faster:
+	$(MAKE) -C faster FASTER_RECURSIVE_MAKE=1
+endif
+
 # process_install_manifest needs to be invoked with --no-remove when building
 # js as standalone because automated builds are building nspr separately and
 # that would remove the resulting files.
 # Eventually, a standalone js build would just be able to build nspr itself,
 # removing the need for the former.
 ifdef JS_STANDALONE
 NO_REMOVE=1
 endif
 
 .PHONY: $(addprefix install-,$(subst /,_,$(install_manifests)))
 $(addprefix install-,$(install_manifests)): install-%: $(install_manifest_depends)
+ifneq (,$(filter FasterMake+RecursiveMake,$(BUILD_BACKENDS)))
+	@# If we're using the hybrid FasterMake/RecursiveMake backend, we want
+	@# to ensure the FasterMake end doesn't have install manifests for the
+	@# same directory, because that would blow up
+	$(if $(wildcard _build_manifests/install/$(subst /,_,$*)),$(if $(wildcard faster/install_$(subst /,_,$*)*),$(error FasterMake and RecursiveMake ends of the hybrid build system want to handle $*)))
+endif
 	$(addprefix $(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )$*) ,$(wildcard _build_manifests/install/$(subst /,_,$*)))
 
 # Dummy wrapper rule to allow the faster backend to piggy back
 install-dist_%: install-dist/% ;
 
 # For compatibility
 .PHONY: install-tests
 install-tests: install-_tests
--- a/config/faster/rules.mk
+++ b/config/faster/rules.mk
@@ -42,24 +42,29 @@ ifndef TEST_MOZBUILD
 default: $(TOPOBJDIR)/dist/bin/platform.ini
 endif
 
 ifndef NO_XPIDL
 # Targets from the recursive make backend to be built for a default build
 default: $(TOPOBJDIR)/config/makefiles/xpidl/xpidl
 endif
 
-ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT))
 # Mac builds require to copy things in dist/bin/*.app
 # TODO: remove the MOZ_WIDGET_TOOLKIT and MOZ_BUILD_APP variables from
 # faster/Makefile and python/mozbuild/mozbuild/test/backend/test_build.py
 # when this is not required anymore.
+# We however don't need to do this when using the hybrid
+# FasterMake/RecursiveMake backend (FASTER_RECURSIVE_MAKE is set when
+# recursing from the RecursiveMake backend)
+ifndef FASTER_RECURSIVE_MAKE
+ifeq (cocoa,$(MOZ_WIDGET_TOOLKIT))
 default:
 	$(MAKE) -C $(TOPOBJDIR)/$(MOZ_BUILD_APP)/app repackage
 endif
+endif
 
 .PHONY: FORCE
 
 # Extra define to trigger some workarounds. We should strive to limit the
 # use of those. As of writing the only ones are in
 # toolkit/content/buildconfig.html and browser/locales/jar.mn.
 ACDEFINES += -DBUILD_FASTER
 
--- a/python/mozbuild/mozbuild/backend/__init__.py
+++ b/python/mozbuild/mozbuild/backend/__init__.py
@@ -3,17 +3,23 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 backends = {
     'AndroidEclipse': 'mozbuild.backend.android_eclipse',
     'ChromeMap': 'mozbuild.codecoverage.chrome_map',
     'CompileDB': 'mozbuild.compilation.database',
     'CppEclipse': 'mozbuild.backend.cpp_eclipse',
     'FasterMake': 'mozbuild.backend.fastermake',
+    'FasterMake+RecursiveMake': None,
     'RecursiveMake': 'mozbuild.backend.recursivemake',
     'VisualStudio': 'mozbuild.backend.visualstudio',
 }
 
 
 def get_backend_class(name):
+    if '+' in name:
+        from mozbuild.backend.base import HybridBackend
+        return HybridBackend(*(get_backend_class(name)
+                               for name in name.split('+')))
+
     class_name = '%sBackend' % name
     module = __import__(backends[name], globals(), locals(), [class_name])
     return getattr(module, class_name)
--- a/python/mozbuild/mozbuild/backend/fastermake.py
+++ b/python/mozbuild/mozbuild/backend/fastermake.py
@@ -121,17 +121,19 @@ class FasterMakeBackend(CommonBackend, P
 
         # Add a few necessary variables inherited from configure
         for var in (
             'PYTHON',
             'ACDEFINES',
             'MOZ_BUILD_APP',
             'MOZ_WIDGET_TOOLKIT',
         ):
-            mk.add_statement('%s = %s' % (var, self.environment.substs[var]))
+            value = self.environment.substs.get(var)
+            if value is not None:
+                mk.add_statement('%s = %s' % (var, value))
 
         install_manifests_bases = self._install_manifests.keys()
 
         # Add information for chrome manifest generation
         manifest_targets = []
 
         for target, entries in self._manifest_entries.iteritems():
             manifest_targets.append(target)
--- a/python/mozbuild/mozbuild/test/backend/test_build.py
+++ b/python/mozbuild/mozbuild/test/backend/test_build.py
@@ -7,16 +7,17 @@ from __future__ import unicode_literals,
 import buildconfig
 import os
 import shutil
 import sys
 import unittest
 import mozpack.path as mozpath
 from contextlib import contextmanager
 from mozunit import main
+from mozbuild.backend import get_backend_class
 from mozbuild.backend.configenvironment import ConfigEnvironment
 from mozbuild.backend.recursivemake import RecursiveMakeBackend
 from mozbuild.backend.fastermake import FasterMakeBackend
 from mozbuild.base import MozbuildObject
 from mozbuild.frontend.emitter import TreeMetadataEmitter
 from mozbuild.frontend.reader import BuildReader
 from mozbuild.util import ensureParentDir
 from mozpack.files import FileFinder
@@ -87,16 +88,40 @@ class TestBuild(unittest.TestCase):
                 'TEST_MOZBUILD=1',
             ]
             with self.line_handler() as handle_make_line:
                 build._run_make(directory=config.topobjdir, target=overrides,
                                 silent=False, line_handler=handle_make_line)
 
             self.validate(config)
 
+    def test_faster_recursive_make(self):
+        substs = list(BASE_SUBSTS) + [
+            ('BUILD_BACKENDS', 'FasterMake+RecursiveMake'),
+        ]
+        with self.do_test_backend(get_backend_class(
+                'FasterMake+RecursiveMake'), substs=substs) as config:
+            buildid = mozpath.join(config.topobjdir, 'config', 'buildid')
+            ensureParentDir(buildid)
+            with open(buildid, 'w') as fh:
+                fh.write('20100101012345\n')
+
+            build = MozbuildObject(config.topsrcdir, None, None,
+                                   config.topobjdir)
+            overrides = [
+                'install_manifest_depends=',
+                'MOZ_CHROME_FILE_FORMAT=flat',
+                'TEST_MOZBUILD=1',
+            ]
+            with self.line_handler() as handle_make_line:
+                build._run_make(directory=config.topobjdir, target=overrides,
+                                silent=False, line_handler=handle_make_line)
+
+            self.validate(config)
+
     def test_faster_make(self):
         substs = list(BASE_SUBSTS) + [
             ('MOZ_BUILD_APP', 'dummy_app'),
             ('MOZ_WIDGET_TOOLKIT', 'dummy_widget'),
         ]
         with self.do_test_backend(RecursiveMakeBackend, FasterMakeBackend,
                                   substs=substs) as config:
             buildid = mozpath.join(config.topobjdir, 'config', 'buildid')