Bug 1454825 - Tup backend: Handle dependent GENERATED_FILES better; r=chmanchester
authorMike Shal <mshal@mozilla.com>
Tue, 17 Apr 2018 19:52:39 -0400
changeset 470489 559cd1e5c775b528fb71ac25c371520a4ee8ec01
parent 470488 18a5d8eaa39deb6dbc1217945bd2ca5d8fffb05e
child 470490 c064e452478eba9d8be71974a79c18fe0dc46acc
push id1728
push userjlund@mozilla.com
push dateMon, 18 Jun 2018 21:12:27 +0000
treeherdermozilla-release@c296fde26f5f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschmanchester
bugs1454825
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1454825 - Tup backend: Handle dependent GENERATED_FILES better; r=chmanchester Work around the fact that tup is bad at handling out-of-order rules by delaying them based on what outputs we've already seen in rule(). MozReview-Commit-ID: G2tyeQr7MTh
python/mozbuild/mozbuild/backend/tup.py
--- a/python/mozbuild/mozbuild/backend/tup.py
+++ b/python/mozbuild/mozbuild/backend/tup.py
@@ -60,16 +60,17 @@ class BackendTupfile(object):
         self.topsrcdir = topsrcdir
         self.objdir = objdir
         self.relobjdir = mozpath.relpath(objdir, topobjdir)
         self.environment = environment
         self.name = mozpath.join(objdir, 'Tupfile')
         self.rules_included = False
         self.defines = []
         self.host_defines = []
+        self.outputs = set()
         self.delayed_generated_files = []
         self.delayed_installed_files = []
         self.per_source_flags = defaultdict(list)
         self.local_flags = defaultdict(list)
         self.sources = defaultdict(list)
         self.host_sources = defaultdict(list)
         self.variables = {}
         self.static_lib = None
@@ -111,16 +112,18 @@ class BackendTupfile(object):
             'inputs': ' '.join(inputs),
             'extra_inputs': ' | ' + ' '.join(extra_inputs) if extra_inputs else '',
             'display': '^%s^ ' % caret_text if caret_text else '',
             'cmd': ' '.join(cmd),
             'outputs': ' '.join(outputs),
             'extra_outputs': ' | ' + ' '.join(extra_outputs) if extra_outputs else '',
         })
 
+        self.outputs.update(outputs)
+
     def symlink_rule(self, source, output=None, output_group=None):
         outputs = [output] if output else [mozpath.basename(source)]
         if output_group:
             outputs.append(output_group)
 
         # The !tup_ln macro does a symlink or file copy (depending on the
         # platform) without shelling out to a subprocess.
         self.rule(
@@ -167,16 +170,28 @@ class BackendTupfile(object):
     def export_shell(self):
         # These are used by mach/mixin/process.py to determine the current
         # shell.
         self.export(['SHELL', 'MOZILLABUILD', 'COMSPEC'])
 
     def close(self):
         return self.fh.close()
 
+    def requires_delay(self, inputs):
+        # We need to delay the generated file rule in the Tupfile until the
+        # generated inputs in the current directory are processed. We do this by
+        # checking all ObjDirPaths to make sure they are in
+        # self.outputs, or are in other directories.
+        for f in inputs:
+            if (isinstance(f, ObjDirPath) and
+                f.target_basename not in self.outputs and
+                mozpath.dirname(f.full_path) == self.objdir):
+                return True
+        return False
+
     @property
     def diff(self):
         return self.fh.diff
 
 
 class TupOnly(CommonBackend, PartialBackend):
     """Backend that generates Tupfiles for the tup build system.
     """
@@ -203,28 +218,16 @@ class TupOnly(CommonBackend, PartialBack
         # The preprocessor including source-repo.h and buildid.h creates
         # dependencies that aren't specified by moz.build and cause errors
         # in Tup. Express these as a group dependency.
         self._early_generated_files = '$(MOZ_OBJ_ROOT)/<early-generated-files>'
 
         self._built_in_addons = set()
         self._built_in_addons_file = 'dist/bin/browser/chrome/browser/content/browser/built_in_addons.json'
 
-        # application.ini.h is a special case since we need to process
-        # the FINAL_TARGET_PP_FILES for application.ini before running
-        # the GENERATED_FILES script, and tup doesn't handle the rules
-        # out of order. Similarly, dependentlibs.list uses libxul as
-        # an input, so must be written after the rule for libxul.
-        self._delayed_files = (
-            'application.ini.h',
-            'dependentlibs.list',
-            'dependentlibs.list.gtest'
-        )
-
-
     def _get_backend_file(self, relobjdir):
         objdir = mozpath.normpath(mozpath.join(self.environment.topobjdir, relobjdir))
         if objdir not in self._backend_files:
             self._backend_files[objdir] = \
                     BackendTupfile(objdir, self.environment,
                                    self.environment.topsrcdir, self.environment.topobjdir)
         return self._backend_files[objdir]
 
@@ -385,17 +388,17 @@ class TupOnly(CommonBackend, PartialBack
 
             if self.environment.is_artifact_build:
                 skip_files = self._compile_env_gen_files
 
             for f in obj.outputs:
                 if any(mozpath.match(f, p) for p in skip_files):
                     return False
 
-            if any([f in obj.outputs for f in self._delayed_files]):
+            if backend_file.requires_delay(obj.inputs):
                 backend_file.delayed_generated_files.append(obj)
             else:
                 self._process_generated_file(backend_file, obj)
         elif (isinstance(obj, ChromeManifestEntry) and
               obj.install_target.startswith('dist/bin')):
             top_level = mozpath.join(obj.install_target, 'chrome.manifest')
             if obj.path != top_level:
                 entry = 'manifest %s' % mozpath.relpath(obj.path,
@@ -460,18 +463,18 @@ class TupOnly(CommonBackend, PartialBack
                                           (backend_file.static_lib and backend_file.static_lib.no_expand_lib,
                                            self._gen_static_library),
                                           (backend_file.program, self._gen_program)):
                 if condition:
                     backend_file.export_shell()
                     gen_method(backend_file)
             for obj in backend_file.delayed_generated_files:
                 self._process_generated_file(backend_file, obj)
-            for path, output in backend_file.delayed_installed_files:
-                backend_file.symlink_rule(path, output=output)
+            for path, output, output_group in backend_file.delayed_installed_files:
+                backend_file.symlink_rule(path, output=output, output_group=output_group)
             with self._write_file(fh=backend_file):
                 pass
 
         with self._write_file(mozpath.join(self.environment.topobjdir, 'Tuprules.tup')) as fh:
             acdefines_flags = ' '.join(['-D%s=%s' % (name, shell_quote(value))
                 for (name, value) in sorted(self.environment.acdefines.iteritems())])
             # TODO: AB_CD only exists in Makefiles at the moment.
             acdefines_flags += ' -DAB_CD=en-US'
@@ -620,18 +623,19 @@ class TupOnly(CommonBackend, PartialBack
                     # We're not generating files in these directories yet, so
                     # don't attempt to install files generated from them.
                     if f.context.relobjdir not in ('layout/style/test',
                                                    'toolkit/library',
                                                    'js/src/shell'):
                         output = mozpath.join('$(MOZ_OBJ_ROOT)', target, path,
                                               f.target_basename)
                         gen_backend_file = self._get_backend_file(f.context.relobjdir)
-                        if f.target_basename in self._delayed_files:
-                            gen_backend_file.delayed_installed_files.append((f.full_path, output))
+                        if gen_backend_file.requires_delay([f]):
+                            output_group = self._installed_files if f.target_basename.endswith('.h') else None
+                            gen_backend_file.delayed_installed_files.append((f.full_path, output, output_group))
                         else:
                             gen_backend_file.symlink_rule(f.full_path, output=output,
                                                           output_group=self._installed_files)
 
 
     def _process_final_target_pp_files(self, obj, backend_file):
         for i, (path, files) in enumerate(obj.files.walk()):
             self._add_features(obj.install_target, path)