Bug 1293448 - Build XPIDL files in the tup backend; r=glandium,gps
authorMike Shal <mshal@mozilla.com>
Fri, 29 Jul 2016 13:43:29 -0400
changeset 312831 5c231de97dec73665c04a3300f44c448fa2c2b42
parent 312830 34767d6a0a2382f3c46723490c122ded0c87386e
child 312832 243fe9c65a048abe71545e206a8a7d05088dd1af
push id32051
push usermshal@mozilla.com
push dateTue, 06 Sep 2016 17:01:01 +0000
treeherderautoland@5c231de97dec [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium, gps
bugs1293448
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1293448 - Build XPIDL files in the tup backend; r=glandium,gps MozReview-Commit-ID: zyojbOFLLn
.gitignore
.hgignore
Makefile.in
moz.configure
python/mozbuild/mozbuild/backend/__init__.py
python/mozbuild/mozbuild/backend/tup.py
--- a/.gitignore
+++ b/.gitignore
@@ -114,8 +114,11 @@ testing/talos/bin/
 testing/talos/include/
 testing/talos/lib/
 testing/talos/talos/tests/tp5n.zip
 testing/talos/talos/tests/tp5n
 testing/talos/talos/tests/devtools/damp.manifest.develop
 
 # Ignore files created when running a reftest.
 lextab.py
+
+# tup database
+/.tup
--- a/.hgignore
+++ b/.hgignore
@@ -124,8 +124,11 @@ GPATH
 ^testing/talos/include/
 ^testing/talos/lib/
 ^testing/talos/talos/tests/tp5n.zip
 ^testing/talos/talos/tests/tp5n
 ^testing/talos/talos/tests/devtools/damp.manifest.develop
 
 # Ignore files created when running a reftest.
 ^lextab.py$
+
+# tup database
+^\.tup
--- a/Makefile.in
+++ b/Makefile.in
@@ -166,16 +166,20 @@ install-manifests: $(addprefix install-,
 # config/faster/rules.mk)
 ifneq (,$(filter FasterMake+RecursiveMake,$(BUILD_BACKENDS)))
 install-manifests: faster
 .PHONY: faster
 faster: install-dist/idl
 	$(MAKE) -C faster FASTER_RECURSIVE_MAKE=1
 endif
 
+.PHONY: tup
+tup: install-manifests buildid.h
+	@$(TUP)
+
 # process_install_manifest needs to be invoked with --no-remove when building
 # js as standalone because automated builds are building nspr separately and
 # that would remove the resulting files.
 # Eventually, a standalone js build would just be able to build nspr itself,
 # removing the need for the former.
 ifdef JS_STANDALONE
 NO_REMOVE=1
 endif
--- a/moz.configure
+++ b/moz.configure
@@ -240,16 +240,27 @@ def possible_makes(make, host):
     if host.kernel == 'WINNT':
         candidates.extend(('make', 'gmake'))
     else:
         candidates.extend(('gmake', 'make'))
     return candidates
 
 check_prog('GMAKE', possible_makes)
 
+# tup detection
+# ==============================================================
+@depends(build_backends)
+def tup_progs(build_backends):
+    for backend in build_backends:
+        if 'Tup' in backend:
+            return ['tup']
+    return None
+
+tup = check_prog('TUP', tup_progs)
+
 # Miscellaneous programs
 # ==============================================================
 check_prog('DOXYGEN', ('doxygen',), allow_missing=True)
 check_prog('XARGS', ('xargs',))
 
 @depends(target)
 def extra_programs(target):
     if target.kernel == 'Darwin':
--- a/python/mozbuild/mozbuild/backend/__init__.py
+++ b/python/mozbuild/mozbuild/backend/__init__.py
@@ -5,16 +5,17 @@
 backends = {
     'AndroidEclipse': 'mozbuild.backend.android_eclipse',
     'ChromeMap': 'mozbuild.codecoverage.chrome_map',
     'CompileDB': 'mozbuild.compilation.database',
     'CppEclipse': 'mozbuild.backend.cpp_eclipse',
     'FasterMake': 'mozbuild.backend.fastermake',
     'FasterMake+RecursiveMake': None,
     'RecursiveMake': 'mozbuild.backend.recursivemake',
+    'Tup': 'mozbuild.backend.tup',
     'VisualStudio': 'mozbuild.backend.visualstudio',
 }
 
 
 def get_backend_class(name):
     if '+' in name:
         from mozbuild.backend.base import HybridBackend
         return HybridBackend(*(get_backend_class(name)
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/backend/tup.py
@@ -0,0 +1,203 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import os
+
+import mozpack.path as mozpath
+from mozbuild.base import MozbuildObject
+from mozbuild.backend.base import PartialBackend, HybridBackend
+from mozbuild.backend.recursivemake import RecursiveMakeBackend
+from mozbuild.shellutil import quote as shell_quote
+
+from .common import CommonBackend
+from ..frontend.data import (
+    ContextDerived,
+)
+from ..util import (
+    FileAvoidWrite,
+)
+
+
+class BackendTupfile(object):
+    """Represents a generated Tupfile.
+    """
+
+    def __init__(self, srcdir, objdir, environment, topsrcdir, topobjdir):
+        self.topsrcdir = topsrcdir
+        self.srcdir = srcdir
+        self.objdir = objdir
+        self.relobjdir = mozpath.relpath(objdir, topobjdir)
+        self.environment = environment
+        self.name = mozpath.join(objdir, 'Tupfile')
+        self.rules_included = False
+
+        self.fh = FileAvoidWrite(self.name, capture_diff=True)
+        self.fh.write('# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT.\n')
+        self.fh.write('\n')
+
+    def write(self, buf):
+        self.fh.write(buf)
+
+    def include_rules(self):
+        if not self.rules_included:
+            self.write('include_rules\n')
+            self.rules_included = True
+
+    def rule(self, cmd, inputs=None, outputs=None, display=None, extra_outputs=None):
+        inputs = inputs or []
+        outputs = outputs or []
+        self.include_rules()
+        self.write(': %(inputs)s |> %(display)s%(cmd)s |> %(outputs)s%(extra_outputs)s\n' % {
+            'inputs': ' '.join(inputs),
+            'display': '^ %s^ ' % display if display else '',
+            'cmd': ' '.join(cmd),
+            'outputs': ' '.join(outputs),
+            'extra_outputs': ' | ' + ' '.join(extra_outputs) if extra_outputs else '',
+        })
+
+    def close(self):
+        return self.fh.close()
+
+    @property
+    def diff(self):
+        return self.fh.diff
+
+
+class TupOnly(CommonBackend, PartialBackend):
+    """Backend that generates Tupfiles for the tup build system.
+    """
+
+    def _init(self):
+        CommonBackend._init(self)
+
+        self._backend_files = {}
+        self._cmd = MozbuildObject.from_environment()
+
+    def _get_backend_file(self, relativedir):
+        objdir = mozpath.join(self.environment.topobjdir, relativedir)
+        srcdir = mozpath.join(self.environment.topsrcdir, relativedir)
+        if objdir not in self._backend_files:
+            self._backend_files[objdir] = \
+                    BackendTupfile(srcdir, objdir, self.environment,
+                                   self.environment.topsrcdir, self.environment.topobjdir)
+        return self._backend_files[objdir]
+
+    def consume_object(self, obj):
+        """Write out build files necessary to build with tup."""
+
+        if not isinstance(obj, ContextDerived):
+            return False
+
+        consumed = CommonBackend.consume_object(self, obj)
+
+        # Even if CommonBackend acknowledged the object, we still need to let
+        # the RecursiveMake backend also handle these objects.
+        if consumed:
+            return False
+
+        return True
+
+    def consume_finished(self):
+        CommonBackend.consume_finished(self)
+
+        for objdir, backend_file in sorted(self._backend_files.items()):
+            with self._write_file(fh=backend_file):
+                pass
+
+        with self._write_file(mozpath.join(self.environment.topobjdir, 'Tuprules.tup')) as fh:
+            acdefines = [name for name in self.environment.defines
+                if not name in self.environment.non_global_defines]
+            acdefines_flags = ' '.join(['-D%s=%s' % (name,
+                shell_quote(self.environment.defines[name]))
+                for name in sorted(acdefines)])
+            fh.write('MOZ_OBJ_ROOT = $(TUP_CWD)\n')
+            fh.write('DIST = $(MOZ_OBJ_ROOT)/dist\n')
+            fh.write('ACDEFINES = %s\n' % acdefines_flags)
+            fh.write('topsrcdir = $(MOZ_OBJ_ROOT)/%s\n' % (
+                os.path.relpath(self.environment.topsrcdir, self.environment.topobjdir)
+            ))
+            fh.write('PYTHON = $(MOZ_OBJ_ROOT)/_virtualenv/bin/python -B\n')
+            fh.write('PYTHON_PATH = $(PYTHON) $(topsrcdir)/config/pythonpath.py\n')
+            fh.write('PLY_INCLUDE = -I$(topsrcdir)/other-licenses/ply\n')
+            fh.write('IDL_PARSER_DIR = $(topsrcdir)/xpcom/idl-parser\n')
+            fh.write('IDL_PARSER_CACHE_DIR = $(MOZ_OBJ_ROOT)/xpcom/idl-parser\n')
+
+        # Run 'tup init' if necessary.
+        if not os.path.exists(mozpath.join(self.environment.topsrcdir, ".tup")):
+            tup = self.environment.substs.get('TUP', 'tup')
+            self._cmd.run_process(cwd=self.environment.topsrcdir, log_name='tup', args=[tup, 'init'])
+
+    def _handle_idl_manager(self, manager):
+
+        # TODO: This should come from GENERATED_FILES, and can be removed once
+        # those are implemented.
+        backend_file = self._get_backend_file('xpcom/idl-parser')
+        backend_file.rule(
+            display='python header.py -> [%o]',
+            cmd=[
+                '$(PYTHON_PATH)',
+                '$(PLY_INCLUDE)',
+                '$(topsrcdir)/xpcom/idl-parser/xpidl/header.py',
+            ],
+            outputs=['xpidlyacc.py', 'xpidllex.py'],
+        )
+
+        backend_file = self._get_backend_file('xpcom/xpidl')
+
+        # These are used by mach/mixin/process.py to determine the current
+        # shell.
+        for var in ('SHELL', 'MOZILLABUILD', 'COMSPEC'):
+            backend_file.write('export %s\n' % var)
+
+        for module, data in sorted(manager.modules.iteritems()):
+            dest, idls = data
+            cmd = [
+                '$(PYTHON_PATH)',
+                '$(PLY_INCLUDE)',
+                '-I$(IDL_PARSER_DIR)',
+                '-I$(IDL_PARSER_CACHE_DIR)',
+                '$(topsrcdir)/python/mozbuild/mozbuild/action/xpidl-process.py',
+                '--cache-dir', '$(MOZ_OBJ_ROOT)/xpcom/idl-parser',
+                '$(DIST)/idl',
+                '$(DIST)/include',
+                '$(MOZ_OBJ_ROOT)/%s/components' % dest,
+                module,
+            ]
+            cmd.extend(sorted(idls))
+
+            outputs = ['$(MOZ_OBJ_ROOT)/%s/components/%s.xpt' % (dest, module)]
+            outputs.extend(['$(MOZ_OBJ_ROOT)/dist/include/%s.h' % f for f in sorted(idls)])
+            backend_file.rule(
+                inputs=[
+                    '$(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidllex.py',
+                    '$(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidlyacc.py',
+                ],
+                display='XPIDL %s' % module,
+                cmd=cmd,
+                outputs=outputs,
+            )
+
+    def _handle_ipdl_sources(self, ipdl_dir, sorted_ipdl_sources,
+                             unified_ipdl_cppsrcs_mapping):
+        # TODO: This isn't implemented yet in the tup backend, but it is called
+        # by the CommonBackend.
+        pass
+
+    def _handle_webidl_build(self, bindings_dir, unified_source_mapping,
+                             webidls, expected_build_output_files,
+                             global_define_files):
+        # TODO: This isn't implemented yet in the tup backend, but it is called
+        # by the CommonBackend.
+        pass
+
+
+class TupBackend(HybridBackend(TupOnly, RecursiveMakeBackend)):
+    def build(self, config, output, jobs, verbose):
+        status = config._run_make(directory=self.environment.topobjdir, target='tup',
+                                  line_handler=output.on_line, log=False, print_directory=False,
+                                  ensure_exit_code=False, num_jobs=jobs, silent=not verbose,
+                                  append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'})
+        return status