Bug 1468547 - Add a default output group to the Tup backend. r=mshal
authorChris Manchester <cmanchester@mozilla.com>
Mon, 09 Jul 2018 14:28:58 -0700
changeset 425543 b9937e7b09956d8228283632aa6d4079c45a0368
parent 425542 cd0eb8dbd0861e34f41e7520dbd21806565c14f1
child 425544 f211edf86a429fd76153b0b3612755ecfb2e1283
push id66125
push usercmanchester@mozilla.com
push dateMon, 09 Jul 2018 21:43:54 +0000
treeherderautoland@b18846ed7d05 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmshal
bugs1468547
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1468547 - Add a default output group to the Tup backend. r=mshal This adds everything to a default output group so we can run `tup upd $objdir/<default>` during `mach build` and `tup upd $objdir/<gtest>` for instance when running `mach gtest` as a way to conditionally build parts of the tree. MozReview-Commit-ID: 8jpkru1EgAC
python/mozbuild/mozbuild/backend/tup.py
--- a/python/mozbuild/mozbuild/backend/tup.py
+++ b/python/mozbuild/mozbuild/backend/tup.py
@@ -66,17 +66,18 @@ from .cargo_build_defs import (
     cargo_extra_flags,
 )
 
 
 class BackendTupfile(object):
     """Represents a generated Tupfile.
     """
 
-    def __init__(self, objdir, environment, topsrcdir, topobjdir, dry_run):
+    def __init__(self, objdir, environment, topsrcdir, topobjdir, dry_run,
+                 default_group):
         self.topsrcdir = topsrcdir
         self.objdir = objdir
         self.relobjdir = mozpath.relpath(objdir, topobjdir)
         self.environment = environment
         self.name = mozpath.join(objdir, 'Tupfile')
         self.rules_included = False
         self.defines = []
         self.host_defines = []
@@ -91,16 +92,18 @@ class BackendTupfile(object):
         self.rust_library = None
         self.static_lib = None
         self.shared_lib = None
         self.programs = []
         self.host_programs = []
         self.host_library = None
         self.exports = set()
 
+        self._default_group = default_group
+
         # These files are special, ignore anything that generates them or
         # depends on them.
         self._skip_files = [
             'built_in_addons.json',
             'signmar',
         ]
 
         self.fh = FileAvoidWrite(self.name, capture_diff=True, dry_run=dry_run)
@@ -111,20 +114,23 @@ class BackendTupfile(object):
         self.fh.write(buf)
 
     def include_rules(self):
         if not self.rules_included:
             self.write('include_rules\n')
             self.rules_included = True
 
     def rule(self, cmd, inputs=None, outputs=None, display=None,
-             extra_inputs=None, extra_outputs=None, check_unchanged=False):
+             extra_inputs=None, output_group=None, check_unchanged=False):
         inputs = inputs or []
         outputs = outputs or []
 
+        if output_group is None:
+            output_group = self._default_group
+
         for f in inputs + outputs:
             if any(f.endswith(skip_file) for skip_file in self._skip_files):
                 return
 
         display = display or ""
         self.include_rules()
         flags = ""
         if check_unchanged:
@@ -133,38 +139,37 @@ class BackendTupfile(object):
             # same.
             flags += "o"
 
         if display:
             caret_text = flags + ' ' + display
         else:
             caret_text = flags
 
-        self.write(': %(inputs)s%(extra_inputs)s |> %(display)s%(cmd)s |> %(outputs)s%(extra_outputs)s\n' % {
+        self.write(': %(inputs)s%(extra_inputs)s |> %(display)s%(cmd)s |> %(outputs)s%(output_group)s\n' % {
             'inputs': ' '.join(inputs),
             'extra_inputs': ' | ' + ' '.join(extra_inputs) if extra_inputs else '',
             'display': '^%s^ ' % caret_text if caret_text else '',
             'cmd': ' '.join(cmd),
             'outputs': ' '.join(outputs),
-            'extra_outputs': ' | ' + ' '.join(extra_outputs) if extra_outputs else '',
+            'output_group': ' ' + output_group if output_group else '',
         })
 
         self.outputs.update(outputs)
 
     def symlink_rule(self, source, output=None, output_group=None):
         outputs = [output] if output else [mozpath.basename(source)]
-        if output_group:
-            outputs.append(output_group)
 
         # The !tup_ln macro does a symlink or file copy (depending on the
         # platform) without shelling out to a subprocess.
         self.rule(
             cmd=['!tup_ln'],
             inputs=[source],
             outputs=outputs,
+            output_group=output_group
         )
 
     def gen_sources_rules(self, extra_inputs):
         sources = self.sources
         host_sources = self.host_sources
         as_dash_c = self.variables.get('AS_DASH_C_FLAG', self.environment.substs['AS_DASH_C_FLAG'])
         compilers = [
             (sources['.S'], 'AS', 'SFLAGS', '-c', ''),
@@ -256,33 +261,34 @@ class TupBackend(CommonBackend):
         # dependencies that aren't specified by moz.build and cause errors
         # in Tup. Express these as a group dependency.
         self._early_generated_files = '$(MOZ_OBJ_ROOT)/<early-generated-files>'
 
         self._built_in_addons = set()
         self._built_in_addons_file = 'dist/bin/browser/chrome/browser/content/browser/built_in_addons.json'
 
         self._shlibs = '$(MOZ_OBJ_ROOT)/<shlibs>'
+        self._default_group = '$(MOZ_OBJ_ROOT)/<default>'
 
     def _get_mozconfig_env(self, config):
         env = {}
         loader = MozconfigLoader(config.topsrcdir)
         mozconfig = loader.read_mozconfig(config.substs['MOZCONFIG'])
         make_extra = mozconfig['make_extra'] or []
         env = {}
         for line in make_extra:
             if line.startswith('export '):
                 line = line[len('export '):]
             key, value = line.split('=')
             env[key] = value
         return env
 
     def build(self, config, output, jobs, verbose, what=None):
         if not what:
-            what = [self.environment.topobjdir]
+            what = ['%s/<default>' % config.topobjdir]
         args = [self.environment.substs['TUP'], 'upd'] + what
         if self.environment.substs.get('MOZ_AUTOMATION'):
             args += ['--quiet']
         if verbose:
             args += ['--verbose']
         if jobs > 0:
             args += ['-j%d' % jobs]
         else:
@@ -300,17 +306,17 @@ class TupBackend(CommonBackend):
         return status
 
     def _get_backend_file(self, relobjdir):
         objdir = mozpath.normpath(mozpath.join(self.environment.topobjdir, relobjdir))
         if objdir not in self._backend_files:
             self._backend_files[objdir] = \
                     BackendTupfile(objdir, self.environment,
                                    self.environment.topsrcdir, self.environment.topobjdir,
-                                   self.dry_run)
+                                   self.dry_run, self._default_group)
         return self._backend_files[objdir]
 
     def _get_backend_file_for(self, obj):
         return self._get_backend_file(obj.relobjdir)
 
     def _py_action(self, action):
         cmd = [
             '$(PYTHON)',
@@ -381,17 +387,17 @@ class TupBackend(CommonBackend):
             [backend_file.environment.substs['OS_LIBS']] +
             os_libs
         )
         backend_file.rule(
             cmd=cmd,
             inputs=inputs,
             extra_inputs=extra_inputs,
             outputs=[shlib.lib_name],
-            extra_outputs=[self._shlibs],
+            output_group=self._shlibs,
             display='LINK %o'
         )
         backend_file.symlink_rule(mozpath.join(backend_file.objdir,
                                                shlib.lib_name),
                                   output=mozpath.join(self.environment.topobjdir,
                                                       shlib.install_target,
                                                       shlib.lib_name),
                                   output_group=self._shlibs)
@@ -783,17 +789,17 @@ class TupBackend(CommonBackend):
 
             invocation['full-deps'] = set(inputs)
             invocation['full-deps'].update(invocation['outputs'])
 
             backend_file.rule(
                 command,
                 inputs=sorted(inputs),
                 outputs=outputs,
-                extra_outputs=[self._rust_libs],
+                output_group=self._rust_libs,
                 extra_inputs=[self._installed_files],
                 display='%s %s' % (header, display_name(invocation)),
             )
 
             for dst, link in invocation['links'].iteritems():
                 backend_file.symlink_rule(link, dst, self._rust_libs)
 
         for val in enumerate(invocations):
@@ -854,19 +860,19 @@ class TupBackend(CommonBackend):
             }
             for f in obj.outputs:
                 exports = extra_exports.get(f)
                 if exports:
                     backend_file.export(exports)
 
             if any(f.endswith(('automation.py', 'source-repo.h', 'buildid.h'))
                    for f in obj.outputs):
-                extra_outputs = [self._early_generated_files]
+                output_group = self._early_generated_files
             else:
-                extra_outputs = [self._installed_files] if obj.required_for_compile else []
+                output_group = self._installed_files if obj.required_for_compile else None
                 full_inputs += [self._early_generated_files]
 
             extra_inputs = []
             if any(f in obj.outputs for f in ('dependentlibs.list',
                                               'dependendentlibs.list.gtest')):
                 extra_inputs += [self._shlibs]
 
             if len(outputs) > 3:
@@ -879,17 +885,17 @@ class TupBackend(CommonBackend):
                 display_outputs=display_outputs
             )
             backend_file.rule(
                 display=display,
                 cmd=cmd,
                 inputs=full_inputs,
                 extra_inputs=extra_inputs,
                 outputs=outputs,
-                extra_outputs=extra_outputs,
+                output_group=output_group,
                 check_unchanged=True,
             )
 
     def _process_defines(self, backend_file, obj, host=False):
         defines = list(obj.get_defines())
         if defines:
             if host:
                 backend_file.host_defines = defines
@@ -1040,17 +1046,17 @@ class TupBackend(CommonBackend):
                 inputs=[
                     '$(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidl/xpidllex.py',
                     '$(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidl/xpidlyacc.py',
                     self._installed_idls,
                 ],
                 display='XPIDL %s' % module,
                 cmd=cmd,
                 outputs=outputs,
-                extra_outputs=[self._installed_files],
+                output_group=self._installed_files,
                 check_unchanged=True,
             )
 
         cpp_backend_file = self._get_backend_file('xpcom/reflect/xptinfo')
         cpp_backend_file.export_shell()
         cpp_backend_file.rule(
             inputs=all_xpts,
             display='XPIDL xptcodegen.py %o',
@@ -1133,17 +1139,17 @@ class TupBackend(CommonBackend):
             for extension in extensions:
                 outputs.append("%s%s.cpp" % (basename, extension))
                 outputs.append(mozpath.join(headerdir, '%s%s.h' % (basename, extension)))
 
         backend_file.rule(
             display='IPDL code generation',
             cmd=cmd,
             outputs=outputs,
-            extra_outputs=[self._installed_files],
+            output_group=self._installed_files,
             check_unchanged=True,
         )
         backend_file.sources['.cpp'].extend(u[0] for u in unified_ipdl_cppsrcs_mapping)
 
     def _handle_webidl_build(self, bindings_dir, unified_source_mapping,
                              webidls, expected_build_output_files,
                              global_define_files):
         backend_file = self._get_backend_file('dom/bindings')
@@ -1166,16 +1172,16 @@ class TupBackend(CommonBackend):
         ]
         outputs.extend(expected_build_output_files)
 
         backend_file.rule(
             display='WebIDL code generation',
             cmd=cmd,
             inputs=webidls.all_non_static_basenames(),
             outputs=outputs,
-            extra_outputs=[self._installed_files],
+            output_group=self._installed_files,
             check_unchanged=True,
         )
         backend_file.sources['.cpp'].extend(u[0] for u in unified_source_mapping)
         backend_file.sources['.cpp'].extend(sorted(global_define_files))
 
         test_backend_file = self._get_backend_file('dom/bindings/test')
         test_backend_file.sources['.cpp'].extend(sorted('../%sBinding.cpp' % s for s in webidls.all_test_stems()))