Bug 1315810 - Use tup's internal symlinking instead of build manifests; r=chmanchester
authorMike Shal <mshal@mozilla.com>
Fri, 30 Sep 2016 11:16:41 -0400
changeset 368325 7a1621d1bea10e135a9c9800f067796cbc02ae9c
parent 368324 bce473db20d7c57241462220e83c5bd735b3d29b
child 368326 cad7e38527b98bbfcc174bea2f00df99d96d1dac
push id6996
push userjlorenzo@mozilla.com
push dateMon, 06 Mar 2017 20:48:21 +0000
treeherdermozilla-beta@d89512dab048 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschmanchester
bugs1315810
milestone53.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1315810 - Use tup's internal symlinking instead of build manifests; r=chmanchester MozReview-Commit-ID: LNsbghIGJvH
Makefile.in
python/mozbuild/mozbuild/backend/tup.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -170,17 +170,17 @@ install-manifests: faster
 faster: install-dist/idl
 	$(MAKE) -C faster FASTER_RECURSIVE_MAKE=1
 endif
 
 .PHONY: tup
 tup:
 	$(call BUILDSTATUS,TIERS make tup)
 	$(call BUILDSTATUS,TIER_START make)
-	$(MAKE) install-manifests buildid.h source-repo.h
+	$(MAKE) buildid.h source-repo.h
 	$(call BUILDSTATUS,TIER_FINISH make)
 	$(call BUILDSTATUS,TIER_START tup)
 	@$(TUP) $(if $(findstring s,$(filter-out --%,$(MAKEFLAGS))),,--verbose)
 	$(call BUILDSTATUS,TIER_FINISH tup)
 
 # process_install_manifest needs to be invoked with --no-remove when building
 # js as standalone because automated builds are building nspr separately and
 # that would remove the resulting files.
--- a/python/mozbuild/mozbuild/backend/tup.py
+++ b/python/mozbuild/mozbuild/backend/tup.py
@@ -75,16 +75,29 @@ class BackendTupfile(object):
         self.write(': %(inputs)s |> %(display)s%(cmd)s |> %(outputs)s%(extra_outputs)s\n' % {
             'inputs': ' '.join(inputs),
             'display': '^%s^ ' % caret_text if caret_text else '',
             'cmd': ' '.join(cmd),
             'outputs': ' '.join(outputs),
             'extra_outputs': ' | ' + ' '.join(extra_outputs) if extra_outputs else '',
         })
 
+    def symlink_rule(self, source, output_group=None):
+        outputs = [mozpath.basename(source)]
+        if output_group:
+            outputs.append(output_group)
+
+        # The !tup_ln macro does a symlink or file copy (depending on the
+        # platform) without shelling out to a subprocess.
+        self.rule(
+            cmd=['!tup_ln'],
+            inputs=[source],
+            outputs=outputs,
+        )
+
     def export_shell(self):
         if not self.shell_exported:
             # These are used by mach/mixin/process.py to determine the current
             # shell.
             for var in ('SHELL', 'MOZILLABUILD', 'COMSPEC'):
                 self.write('export %s\n' % var)
             self.shell_exported = True
 
@@ -101,16 +114,20 @@ class TupOnly(CommonBackend, PartialBack
     """
 
     def _init(self):
         CommonBackend._init(self)
 
         self._backend_files = {}
         self._cmd = MozbuildObject.from_environment()
 
+        # This is a 'group' dependency - All rules that list this as an output
+        # will be built before any rules that list this as an input.
+        self._installed_files = '$(MOZ_OBJ_ROOT)/<installed-files>'
+
     def _get_backend_file(self, relativedir):
         objdir = mozpath.join(self.environment.topobjdir, relativedir)
         srcdir = mozpath.join(self.environment.topsrcdir, relativedir)
         if objdir not in self._backend_files:
             self._backend_files[objdir] = \
                     BackendTupfile(srcdir, objdir, self.environment,
                                    self.environment.topsrcdir, self.environment.topobjdir)
         return self._backend_files[objdir]
@@ -128,21 +145,18 @@ class TupOnly(CommonBackend, PartialBack
 
     def consume_object(self, obj):
         """Write out build files necessary to build with tup."""
 
         if not isinstance(obj, ContextDerived):
             return False
 
         consumed = CommonBackend.consume_object(self, obj)
-
-        # Even if CommonBackend acknowledged the object, we still need to let
-        # the RecursiveMake backend also handle these objects.
         if consumed:
-            return False
+            return True
 
         backend_file = self._get_backend_file_for(obj)
 
         if isinstance(obj, GeneratedFile):
             # These files are already generated by make before tup runs.
             skip_files = (
                 'buildid.h',
                 'source-repo.h',
@@ -245,16 +259,20 @@ class TupOnly(CommonBackend, PartialBack
 
     def _process_final_target_pp_files(self, obj, backend_file):
         for i, (path, files) in enumerate(obj.files.walk()):
             for f in files:
                 self._preprocess(backend_file, f.full_path,
                                  destdir=mozpath.join(self.environment.topobjdir, obj.install_target, path))
 
     def _handle_idl_manager(self, manager):
+        dist_idl_backend_file = self._get_backend_file('dist/idl')
+        for idl in manager.idls.values():
+            dist_idl_backend_file.symlink_rule(idl['source'], output_group=self._installed_files)
+
         backend_file = self._get_backend_file('xpcom/xpidl')
         backend_file.export_shell()
 
         for module, data in sorted(manager.modules.iteritems()):
             dest, idls = data
             cmd = [
                 '$(PYTHON_PATH)',
                 '$(PLY_INCLUDE)',
@@ -270,16 +288,17 @@ class TupOnly(CommonBackend, PartialBack
             cmd.extend(sorted(idls))
 
             outputs = ['$(MOZ_OBJ_ROOT)/%s/components/%s.xpt' % (dest, module)]
             outputs.extend(['$(MOZ_OBJ_ROOT)/dist/include/%s.h' % f for f in sorted(idls)])
             backend_file.rule(
                 inputs=[
                     '$(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidl/xpidllex.py',
                     '$(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidl/xpidlyacc.py',
+                    self._installed_files,
                 ],
                 display='XPIDL %s' % module,
                 cmd=cmd,
                 outputs=outputs,
             )
 
     def _preprocess(self, backend_file, input_file, destdir=None):
         cmd = self._py_action('preprocessor')