Bug 1293448 - Build XPIDL files in the tup backend; r=glandium,gps
authorMike Shal <mshal@mozilla.com>
Fri, 29 Jul 2016 13:43:29 -0400
changeset 312866 5c231de97dec73665c04a3300f44c448fa2c2b42
parent 312865 34767d6a0a2382f3c46723490c122ded0c87386e
child 312867 243fe9c65a048abe71545e206a8a7d05088dd1af
push id20470
push usercbook@mozilla.com
push dateWed, 07 Sep 2016 15:25:31 +0000
treeherderfx-team@95acb9299faf [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium, gps
bugs1293448
milestone51.0a1
Bug 1293448 - Build XPIDL files in the tup backend; r=glandium,gps MozReview-Commit-ID: zyojbOFLLn
.gitignore
.hgignore
Makefile.in
moz.configure
python/mozbuild/mozbuild/backend/__init__.py
python/mozbuild/mozbuild/backend/tup.py
--- a/.gitignore
+++ b/.gitignore
@@ -114,8 +114,11 @@ testing/talos/bin/
 testing/talos/include/
 testing/talos/lib/
 testing/talos/talos/tests/tp5n.zip
 testing/talos/talos/tests/tp5n
 testing/talos/talos/tests/devtools/damp.manifest.develop
 
 # Ignore files created when running a reftest.
 lextab.py
+
+# tup database
+/.tup
--- a/.hgignore
+++ b/.hgignore
@@ -124,8 +124,11 @@ GPATH
 ^testing/talos/include/
 ^testing/talos/lib/
 ^testing/talos/talos/tests/tp5n.zip
 ^testing/talos/talos/tests/tp5n
 ^testing/talos/talos/tests/devtools/damp.manifest.develop
 
 # Ignore files created when running a reftest.
 ^lextab.py$
+
+# tup database
+^\.tup
--- a/Makefile.in
+++ b/Makefile.in
@@ -166,16 +166,20 @@ install-manifests: $(addprefix install-,
 # config/faster/rules.mk)
 ifneq (,$(filter FasterMake+RecursiveMake,$(BUILD_BACKENDS)))
 install-manifests: faster
 .PHONY: faster
 faster: install-dist/idl
 	$(MAKE) -C faster FASTER_RECURSIVE_MAKE=1
 endif
 
+.PHONY: tup
+tup: install-manifests buildid.h
+	@$(TUP)
+
 # process_install_manifest needs to be invoked with --no-remove when building
 # js as standalone because automated builds are building nspr separately and
 # that would remove the resulting files.
 # Eventually, a standalone js build would just be able to build nspr itself,
 # removing the need for the former.
 ifdef JS_STANDALONE
 NO_REMOVE=1
 endif
--- a/moz.configure
+++ b/moz.configure
@@ -240,16 +240,27 @@ def possible_makes(make, host):
     if host.kernel == 'WINNT':
         candidates.extend(('make', 'gmake'))
     else:
         candidates.extend(('gmake', 'make'))
     return candidates
 
 check_prog('GMAKE', possible_makes)
 
+# tup detection
+# ==============================================================
+@depends(build_backends)
+def tup_progs(build_backends):
+    for backend in build_backends:
+        if 'Tup' in backend:
+            return ['tup']
+    return None
+
+tup = check_prog('TUP', tup_progs)
+
 # Miscellaneous programs
 # ==============================================================
 check_prog('DOXYGEN', ('doxygen',), allow_missing=True)
 check_prog('XARGS', ('xargs',))
 
 @depends(target)
 def extra_programs(target):
     if target.kernel == 'Darwin':
--- a/python/mozbuild/mozbuild/backend/__init__.py
+++ b/python/mozbuild/mozbuild/backend/__init__.py
@@ -5,16 +5,17 @@
 backends = {
     'AndroidEclipse': 'mozbuild.backend.android_eclipse',
     'ChromeMap': 'mozbuild.codecoverage.chrome_map',
     'CompileDB': 'mozbuild.compilation.database',
     'CppEclipse': 'mozbuild.backend.cpp_eclipse',
     'FasterMake': 'mozbuild.backend.fastermake',
     'FasterMake+RecursiveMake': None,
     'RecursiveMake': 'mozbuild.backend.recursivemake',
+    'Tup': 'mozbuild.backend.tup',
     'VisualStudio': 'mozbuild.backend.visualstudio',
 }
 
 
 def get_backend_class(name):
     if '+' in name:
         from mozbuild.backend.base import HybridBackend
         return HybridBackend(*(get_backend_class(name)
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/backend/tup.py
@@ -0,0 +1,203 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import os
+
+import mozpack.path as mozpath
+from mozbuild.base import MozbuildObject
+from mozbuild.backend.base import PartialBackend, HybridBackend
+from mozbuild.backend.recursivemake import RecursiveMakeBackend
+from mozbuild.shellutil import quote as shell_quote
+
+from .common import CommonBackend
+from ..frontend.data import (
+    ContextDerived,
+)
+from ..util import (
+    FileAvoidWrite,
+)
+
+
+class BackendTupfile(object):
+    """Represents a generated Tupfile.
+    """
+
+    def __init__(self, srcdir, objdir, environment, topsrcdir, topobjdir):
+        self.topsrcdir = topsrcdir
+        self.srcdir = srcdir
+        self.objdir = objdir
+        self.relobjdir = mozpath.relpath(objdir, topobjdir)
+        self.environment = environment
+        self.name = mozpath.join(objdir, 'Tupfile')
+        self.rules_included = False
+
+        self.fh = FileAvoidWrite(self.name, capture_diff=True)
+        self.fh.write('# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT.\n')
+        self.fh.write('\n')
+
+    def write(self, buf):
+        self.fh.write(buf)
+
+    def include_rules(self):
+        if not self.rules_included:
+            self.write('include_rules\n')
+            self.rules_included = True
+
+    def rule(self, cmd, inputs=None, outputs=None, display=None, extra_outputs=None):
+        inputs = inputs or []
+        outputs = outputs or []
+        self.include_rules()
+        self.write(': %(inputs)s |> %(display)s%(cmd)s |> %(outputs)s%(extra_outputs)s\n' % {
+            'inputs': ' '.join(inputs),
+            'display': '^ %s^ ' % display if display else '',
+            'cmd': ' '.join(cmd),
+            'outputs': ' '.join(outputs),
+            'extra_outputs': ' | ' + ' '.join(extra_outputs) if extra_outputs else '',
+        })
+
+    def close(self):
+        return self.fh.close()
+
+    @property
+    def diff(self):
+        return self.fh.diff
+
+
+class TupOnly(CommonBackend, PartialBackend):
+    """Backend that generates Tupfiles for the tup build system.
+    """
+
+    def _init(self):
+        CommonBackend._init(self)
+
+        self._backend_files = {}
+        self._cmd = MozbuildObject.from_environment()
+
+    def _get_backend_file(self, relativedir):
+        objdir = mozpath.join(self.environment.topobjdir, relativedir)
+        srcdir = mozpath.join(self.environment.topsrcdir, relativedir)
+        if objdir not in self._backend_files:
+            self._backend_files[objdir] = \
+                    BackendTupfile(srcdir, objdir, self.environment,
+                                   self.environment.topsrcdir, self.environment.topobjdir)
+        return self._backend_files[objdir]
+
+    def consume_object(self, obj):
+        """Write out build files necessary to build with tup."""
+
+        if not isinstance(obj, ContextDerived):
+            return False
+
+        consumed = CommonBackend.consume_object(self, obj)
+
+        # Even if CommonBackend acknowledged the object, we still need to let
+        # the RecursiveMake backend also handle these objects.
+        if consumed:
+            return False
+
+        return True
+
+    def consume_finished(self):
+        CommonBackend.consume_finished(self)
+
+        for objdir, backend_file in sorted(self._backend_files.items()):
+            with self._write_file(fh=backend_file):
+                pass
+
+        with self._write_file(mozpath.join(self.environment.topobjdir, 'Tuprules.tup')) as fh:
+            acdefines = [name for name in self.environment.defines
+                if not name in self.environment.non_global_defines]
+            acdefines_flags = ' '.join(['-D%s=%s' % (name,
+                shell_quote(self.environment.defines[name]))
+                for name in sorted(acdefines)])
+            fh.write('MOZ_OBJ_ROOT = $(TUP_CWD)\n')
+            fh.write('DIST = $(MOZ_OBJ_ROOT)/dist\n')
+            fh.write('ACDEFINES = %s\n' % acdefines_flags)
+            fh.write('topsrcdir = $(MOZ_OBJ_ROOT)/%s\n' % (
+                os.path.relpath(self.environment.topsrcdir, self.environment.topobjdir)
+            ))
+            fh.write('PYTHON = $(MOZ_OBJ_ROOT)/_virtualenv/bin/python -B\n')
+            fh.write('PYTHON_PATH = $(PYTHON) $(topsrcdir)/config/pythonpath.py\n')
+            fh.write('PLY_INCLUDE = -I$(topsrcdir)/other-licenses/ply\n')
+            fh.write('IDL_PARSER_DIR = $(topsrcdir)/xpcom/idl-parser\n')
+            fh.write('IDL_PARSER_CACHE_DIR = $(MOZ_OBJ_ROOT)/xpcom/idl-parser\n')
+
+        # Run 'tup init' if necessary.
+        if not os.path.exists(mozpath.join(self.environment.topsrcdir, ".tup")):
+            tup = self.environment.substs.get('TUP', 'tup')
+            self._cmd.run_process(cwd=self.environment.topsrcdir, log_name='tup', args=[tup, 'init'])
+
+    def _handle_idl_manager(self, manager):
+
+        # TODO: This should come from GENERATED_FILES, and can be removed once
+        # those are implemented.
+        backend_file = self._get_backend_file('xpcom/idl-parser')
+        backend_file.rule(
+            display='python header.py -> [%o]',
+            cmd=[
+                '$(PYTHON_PATH)',
+                '$(PLY_INCLUDE)',
+                '$(topsrcdir)/xpcom/idl-parser/xpidl/header.py',
+            ],
+            outputs=['xpidlyacc.py', 'xpidllex.py'],
+        )
+
+        backend_file = self._get_backend_file('xpcom/xpidl')
+
+        # These are used by mach/mixin/process.py to determine the current
+        # shell.
+        for var in ('SHELL', 'MOZILLABUILD', 'COMSPEC'):
+            backend_file.write('export %s\n' % var)
+
+        for module, data in sorted(manager.modules.iteritems()):
+            dest, idls = data
+            cmd = [
+                '$(PYTHON_PATH)',
+                '$(PLY_INCLUDE)',
+                '-I$(IDL_PARSER_DIR)',
+                '-I$(IDL_PARSER_CACHE_DIR)',
+                '$(topsrcdir)/python/mozbuild/mozbuild/action/xpidl-process.py',
+                '--cache-dir', '$(MOZ_OBJ_ROOT)/xpcom/idl-parser',
+                '$(DIST)/idl',
+                '$(DIST)/include',
+                '$(MOZ_OBJ_ROOT)/%s/components' % dest,
+                module,
+            ]
+            cmd.extend(sorted(idls))
+
+            outputs = ['$(MOZ_OBJ_ROOT)/%s/components/%s.xpt' % (dest, module)]
+            outputs.extend(['$(MOZ_OBJ_ROOT)/dist/include/%s.h' % f for f in sorted(idls)])
+            backend_file.rule(
+                inputs=[
+                    '$(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidllex.py',
+                    '$(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidlyacc.py',
+                ],
+                display='XPIDL %s' % module,
+                cmd=cmd,
+                outputs=outputs,
+            )
+
+    def _handle_ipdl_sources(self, ipdl_dir, sorted_ipdl_sources,
+                             unified_ipdl_cppsrcs_mapping):
+        # TODO: This isn't implemented yet in the tup backend, but it is called
+        # by the CommonBackend.
+        pass
+
+    def _handle_webidl_build(self, bindings_dir, unified_source_mapping,
+                             webidls, expected_build_output_files,
+                             global_define_files):
+        # TODO: This isn't implemented yet in the tup backend, but it is called
+        # by the CommonBackend.
+        pass
+
+
+class TupBackend(HybridBackend(TupOnly, RecursiveMakeBackend)):
+    def build(self, config, output, jobs, verbose):
+        status = config._run_make(directory=self.environment.topobjdir, target='tup',
+                                  line_handler=output.on_line, log=False, print_directory=False,
+                                  ensure_exit_code=False, num_jobs=jobs, silent=not verbose,
+                                  append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'})
+        return status