Bug 1454771 - Move tup invocation from Makefile.in to mach. r=mshal
authorChris Manchester <cmanchester@mozilla.com>
Wed, 25 Apr 2018 14:50:18 -0700
changeset 415603 dee93aadc7215d19a1c75ea00ff05037755d1e0b
parent 415602 d9667fd69b462d71df6fd02a17cecc3fcb8c44ca
child 415604 fc793911d371a449b30a81575e5e9a39c6f4f072
push id33900
push userdluca@mozilla.com
push dateThu, 26 Apr 2018 04:51:04 +0000
treeherdermozilla-central@76f35d0ecaa6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmshal
bugs1454771
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1454771 - Move tup invocation from Makefile.in to mach. r=mshal MozReview-Commit-ID: HkhK4oe93Vm
Makefile.in
python/mozbuild/mozbuild/backend/base.py
python/mozbuild/mozbuild/backend/tup.py
python/mozbuild/mozbuild/controller/building.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -144,28 +144,16 @@ install-manifests: $(addprefix install-,
 # config/faster/rules.mk)
 ifneq (,$(filter FasterMake+RecursiveMake,$(BUILD_BACKENDS)))
 install-manifests: faster
 .PHONY: faster
 faster: install-dist/idl
 	$(MAKE) -C faster FASTER_RECURSIVE_MAKE=1
 endif
 
-.PHONY: tup
-tup:
-	$(call BUILDSTATUS,TIERS $(if $(MOZ_ARTIFACT_BUILDS),artifact )tup)
-ifdef MOZ_ARTIFACT_BUILDS
-	$(call BUILDSTATUS,TIER_START artifact)
-	$(MAKE) recurse_artifact
-	$(call BUILDSTATUS,TIER_FINISH artifact)
-endif
-	$(call BUILDSTATUS,TIER_START tup)
-	@$(TUP) $(if $(MOZ_AUTOMATION),--quiet) $(if $(findstring s,$(filter-out --%,$(MAKEFLAGS))),,--verbose) $(topobjdir)
-	$(call BUILDSTATUS,TIER_FINISH tup)
-
 .PHONY: $(addprefix install-,$(install_manifests))
 $(addprefix install-,$(install_manifests)): install-%: $(install_manifest_depends)
 ifneq (,$(filter FasterMake+RecursiveMake,$(BUILD_BACKENDS)))
 	@# If we're using the hybrid FasterMake/RecursiveMake backend, we want
 	@# to ensure the FasterMake end doesn't have install manifests for the
 	@# same directory, because that would blow up
 	$(if $(wildcard _build_manifests/install/$(subst /,_,$*)),$(if $(wildcard faster/install_$(subst /,_,$*)*),$(error FasterMake and RecursiveMake ends of the hybrid build system want to handle $*)))
 endif
--- a/python/mozbuild/mozbuild/backend/base.py
+++ b/python/mozbuild/mozbuild/backend/base.py
@@ -189,17 +189,17 @@ class BuildBackend(LoggingMixin):
 
         This is the main method used by child classes to react to build
         metadata.
         """
 
     def consume_finished(self):
         """Called when consume() has completed handling all objects."""
 
-    def build(self, config, output, jobs, verbose):
+    def build(self, config, output, jobs, verbose, what=None):
         """Called when 'mach build' is executed.
 
         This should return the status value of a subprocess, where 0 denotes
         success and any other value is an error code. A return value of None
         indicates that the default 'make -f client.mk' should run.
         """
         return None
 
--- a/python/mozbuild/mozbuild/backend/tup.py
+++ b/python/mozbuild/mozbuild/backend/tup.py
@@ -10,27 +10,29 @@ import sys
 
 import mozpack.path as mozpath
 from mozbuild.base import MozbuildObject
 from mozbuild.backend.base import PartialBackend, HybridBackend
 from mozbuild.backend.recursivemake import RecursiveMakeBackend
 from mozbuild.shellutil import quote as shell_quote
 from mozbuild.util import OrderedDefaultDict
 from collections import defaultdict
+import multiprocessing
 
 from mozpack.files import (
     FileFinder,
 )
 
 from .common import CommonBackend
 from ..frontend.data import (
     ChromeManifestEntry,
     ComputedFlags,
     ContextDerived,
     Defines,
+    DirectoryTraversal,
     FinalTargetFiles,
     FinalTargetPreprocessedFiles,
     GeneratedFile,
     GeneratedSources,
     HostDefines,
     HostSources,
     JARManifest,
     ObjdirFiles,
@@ -187,17 +189,17 @@ class BackendTupfile(object):
                 return True
         return False
 
     @property
     def diff(self):
         return self.fh.diff
 
 
-class TupOnly(CommonBackend, PartialBackend):
+class TupBackend(CommonBackend):
     """Backend that generates Tupfiles for the tup build system.
     """
 
     def _init(self):
         CommonBackend._init(self)
 
         self._backend_files = {}
         self._cmd = MozbuildObject.from_environment()
@@ -218,16 +220,32 @@ class TupOnly(CommonBackend, PartialBack
         # The preprocessor including source-repo.h and buildid.h creates
         # dependencies that aren't specified by moz.build and cause errors
         # in Tup. Express these as a group dependency.
         self._early_generated_files = '$(MOZ_OBJ_ROOT)/<early-generated-files>'
 
         self._built_in_addons = set()
         self._built_in_addons_file = 'dist/bin/browser/chrome/browser/content/browser/built_in_addons.json'
 
+    def build(self, config, output, jobs, verbose, what=None):
+        if not what:
+            what = [self.environment.topobjdir]
+        args = [self.environment.substs['TUP'], 'upd'] + what
+        if self.environment.substs.get('MOZ_AUTOMATION'):
+            args += ['--quiet']
+        if verbose:
+            args += ['--verbose']
+        if jobs > 0:
+            args += ['-j%d' % jobs]
+        else:
+            args += ['-j%d' % multiprocessing.cpu_count()]
+        return config.run_process(args=args,
+                                  line_handler=output.on_line,
+                                  ensure_exit_code=False)
+
     def _get_backend_file(self, relobjdir):
         objdir = mozpath.normpath(mozpath.join(self.environment.topobjdir, relobjdir))
         if objdir not in self._backend_files:
             self._backend_files[objdir] = \
                     BackendTupfile(objdir, self.environment,
                                    self.environment.topsrcdir, self.environment.topobjdir)
         return self._backend_files[objdir]
 
@@ -428,22 +446,18 @@ class TupOnly(CommonBackend, PartialBack
         elif isinstance(obj, StaticLibrary):
             backend_file.static_lib = obj
         elif isinstance(obj, SharedLibrary):
             backend_file.shared_lib = obj
         elif isinstance(obj, HostProgram):
             pass
         elif isinstance(obj, Program):
             backend_file.program = obj
-
-        # The top-level Makefile.in still contains our driver target and some
-        # things related to artifact builds, so as a special case ensure the
-        # make backend generates a Makefile there.
-        if obj.objdir == self.environment.topobjdir:
-            return False
+        elif isinstance(obj, DirectoryTraversal):
+            pass
 
         return True
 
     def consume_finished(self):
         CommonBackend.consume_finished(self)
 
         # The approach here is similar to fastermake.py, but we
         # simply write out the resulting files here.
@@ -823,16 +837,8 @@ class TupOnly(CommonBackend, PartialBack
             extra_outputs=[self._installed_files],
             check_unchanged=True,
         )
         backend_file.sources['.cpp'].extend(u[0] for u in unified_source_mapping)
         backend_file.sources['.cpp'].extend(sorted(global_define_files))
 
         test_backend_file = self._get_backend_file('dom/bindings/test')
         test_backend_file.sources['.cpp'].extend(sorted('../%sBinding.cpp' % s for s in webidls.all_test_stems()))
-
-
-class TupBackend(HybridBackend(TupOnly, RecursiveMakeBackend)):
-    def build(self, config, output, jobs, verbose):
-        status = config._run_make(directory=self.environment.topobjdir, target='tup',
-                                  line_handler=output.on_line, log=False, print_directory=False,
-                                  ensure_exit_code=False, num_jobs=jobs, silent=not verbose)
-        return status
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -965,25 +965,74 @@ class BuildDriver(MozbuildObject):
                 return 1
 
             if directory is not None:
                 disable_extra_make_dependencies=True
                 directory = mozpath.normsep(directory)
                 if directory.startswith('/'):
                     directory = directory[1:]
 
-            status = None
+            def backend_out_of_date(backend_file):
+                dep_file = '%s.in' % backend_file
+                if not os.path.exists(backend_file):
+                    return True
+                if not os.path.exists(dep_file):
+                    return True
+
+                dep_files = None
+                with open(dep_file, 'r') as fh:
+                    dep_files = fh.read().splitlines()
+                if not dep_files:
+                    return True
+
+                mtime = os.path.getmtime(backend_file)
+                for f in dep_files:
+                    if os.path.getmtime(f) > mtime:
+                        return True
+
+                return False
+
+            def maybe_invoke_backend(active_backend):
+                # Attempt to bypass the make-oriented logic below. Note this
+                # will only succeed in case we're building with a non-make
+                # backend (Tup), and otherwise be harmless.
+                if active_backend:
+                    if 'Make' in active_backend:
+                        return None
+                    if backend_out_of_date(mozpath.join(self.topobjdir,
+                                                        'backend.%sBackend' %
+                                                        active_backend)):
+                        print('Build configuration changed. Regenerating backend.')
+                        args = [config.substs['PYTHON'],
+                                mozpath.join(self.topobjdir, 'config.status')]
+                        self.run_process(args, cwd=self.topobjdir)
+                    backend_cls = get_backend_class(active_backend)(config)
+                    return backend_cls.build(self, output, jobs, verbose, what)
+                return None
+
             monitor.start_resource_recording()
-            if what:
-                top_make = os.path.join(self.topobjdir, 'Makefile')
-                if not os.path.exists(top_make):
-                    print('Your tree has not been configured yet. Please run '
-                        '|mach build| with no arguments.')
-                    return 1
+
+            config = None
+            try:
+                config = self.config_environment
+            except Exception:
+                pass
 
+            if config is None:
+                config_rc = self.configure(buildstatus_messages=True,
+                                           line_handler=output.on_line)
+                if config_rc != 0:
+                    return config_rc
+
+                config = self.config_environment
+
+            status = maybe_invoke_backend(config.substs.get('BUILD_BACKENDS',
+                                                            [None])[0])
+
+            if what and status is None:
                 # Collect target pairs.
                 target_pairs = []
                 for target in what:
                     path_arg = self._wrap_path_argument(target)
 
                     if directory is not None:
                         make_dir = os.path.join(self.topobjdir, directory)
                         make_target = target
@@ -1042,55 +1091,28 @@ class BuildDriver(MozbuildObject):
                     status = self._run_make(directory=make_dir, target=make_target,
                         line_handler=output.on_line, log=False, print_directory=False,
                         ensure_exit_code=False, num_jobs=jobs, silent=not verbose,
                         append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'},
                         keep_going=keep_going)
 
                     if status != 0:
                         break
-            else:
-                # Try to call the default backend's build() method. This will
-                # run configure to determine BUILD_BACKENDS if it hasn't run
-                # yet.
-                config = None
-                try:
-                    config = self.config_environment
-                except Exception:
-                    config_rc = self.configure(buildstatus_messages=True,
-                                               line_handler=output.on_line)
-                    if config_rc != 0:
-                        return config_rc
 
-                    # Even if configure runs successfully, we may have trouble
-                    # getting the config_environment for some builds, such as
-                    # OSX Universal builds. These have to go through client.mk
-                    # regardless.
-                    try:
-                        config = self.config_environment
-                    except Exception:
-                        pass
-
-                if config:
-                    active_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
-                    if active_backend:
-                        backend_cls = get_backend_class(active_backend)(config)
-                        status = backend_cls.build(self, output, jobs, verbose)
-
+            elif status is None:
                 # If the backend doesn't specify a build() method, then just
                 # call client.mk directly.
-                if status is None:
-                    status = self._run_client_mk(line_handler=output.on_line,
-                                                 jobs=jobs,
-                                                 verbose=verbose,
-                                                 keep_going=keep_going)
+                status = self._run_client_mk(line_handler=output.on_line,
+                                             jobs=jobs,
+                                             verbose=verbose,
+                                             keep_going=keep_going)
 
-                self.log(logging.WARNING, 'warning_summary',
-                    {'count': len(monitor.warnings_database)},
-                    '{count} compiler warnings present.')
+            self.log(logging.WARNING, 'warning_summary',
+                     {'count': len(monitor.warnings_database)},
+                     '{count} compiler warnings present.')
 
             # Try to run the active build backend's post-build step, if possible.
             try:
                 config = self.config_environment
                 active_backend = config.substs.get('BUILD_BACKENDS', [None])[0]
                 if active_backend:
                     backend_cls = get_backend_class(active_backend)(config)
                     new_status = backend_cls.post_build(self, output, jobs, verbose, status)