Bug 923080 - Generate xpt files into final location. r=glandium
authorBrian O'Keefe <bokeefe@alum.wpi.edu>
Wed, 21 May 2014 08:25:57 -0400
changeset 246763 ccbd5ea13dd71912f2de6740629e78f85673c5f5
parent 246762 4fc84f5e110fff80f186f5c30d8a8ea2a5374638
child 246764 dd9790020b8953505836f22fd3f189baec75c77b
push id4489
push userraliiev@mozilla.com
push dateMon, 23 Feb 2015 15:17:55 +0000
treeherdermozilla-beta@fd7c3dc24146 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs923080
milestone37.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 923080 - Generate xpt files into final location. r=glandium Now that the mozbuild backend knows about FINAL_TARGET, we are able to install generated xpt files into their final location. This saves us from copying xpt files into their final location on every build. Original patch by gps, rebased and comments addressed by Ms2ger
config/makefiles/xpidl/Makefile.in
config/rules.mk
js/xpconnect/tests/idl/Makefile.in
js/xpconnect/tests/idl/moz.build
python/mozbuild/mozbuild/backend/common.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/context.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
--- a/config/makefiles/xpidl/Makefile.in
+++ b/config/makefiles/xpidl/Makefile.in
@@ -17,54 +17,45 @@ include $(topsrcdir)/config/rules.mk
 #
 # XPIDL files are logically grouped together by modules. The typelib
 # information for all XPIDLs in the same module is linked together into
 # an .xpt file having the name of the module.
 #
 # As an optimization to reduce overall CPU usage, we process all .idl
 # belonging to a module with a single command invocation. This prevents
 # redundant parsing of .idl files and significantly reduces CPU cycles.
-#
-# Future improvement: Headers are currently written to a local directory then
-# installed in the distribution directory. It is preferable to write headers
-# directly into the distribution directory. However, PGO builds remove the dist
-# directory via rm -rf (with no regards to manifests). Since the cost of
-# processing XPIDL files is not trivial, it is preferrable to cache the headers
-# and reinstall them rather than regenerate them. Ideally the dist pruning is
-# performed with manifests. At that time we can write headers directly to the
-# dist directory.
 
 # For dependency files.
 idl_deps_dir := .deps
 
-# Where we put our final, linked .xpt files.
-idl_xpt_dir := xpt
-
 dist_idl_dir := $(DIST)/idl
 dist_include_dir := $(DIST)/include
 process_py := $(topsrcdir)/python/mozbuild/mozbuild/action/xpidl-process.py
 
+ifdef LIBXUL_SDK
+libxul_sdk_includes := -I$(LIBXUL_SDK)/idl
+endif
+
 # TODO we should use py_action, but that would require extra directories to be
 # in the virtualenv.
-idlprocess := $(PYTHON_PATH) $(PLY_INCLUDE) -I$(IDL_PARSER_DIR) -I$(IDL_PARSER_CACHE_DIR) \
-    $(process_py) --cache-dir $(IDL_PARSER_CACHE_DIR) $(dist_idl_dir) \
-        $(dist_include_dir) $(idl_xpt_dir) $(idl_deps_dir)
-
-ifdef LIBXUL_SDK
-idlprocess += -I$(LIBXUL_SDK)/idl
-endif
+%.xpt:
+	@echo "$(@F)"
+	$(PYTHON_PATH) $(PLY_INCLUDE) -I$(IDL_PARSER_DIR) -I$(IDL_PARSER_CACHE_DIR) \
+		$(process_py) --cache-dir $(IDL_PARSER_CACHE_DIR) $(dist_idl_dir) \
+		$(dist_include_dir) $(@D) $(idl_deps_dir) $(libxul_sdk_includes) \
+		$(basename $(notdir $@ $(filter %.idl,$^)))
 
 xpidl_modules := @xpidl_modules@
+xpt_files := @xpt_files@
 
 @xpidl_rules@
 
-linked_xpt_files := $(addprefix $(idl_xpt_dir)/,$(addsuffix .xpt,$(xpidl_modules)))
 depends_files := $(foreach root,$(xpidl_modules),$(idl_deps_dir)/$(root).pp)
 
-GARBAGE += $(linked_xpt_files) $(depends_files)
+GARBAGE += $(xpt_files) $(depends_files)
 
-xpidl:: $(linked_xpt_files)
+xpidl:: $(xpt_files)
 
-$(linked_xpt_files): $(process_py) $(call mkdir_deps,$(idl_deps_dir) $(dist_include_dir) $(idl_xpt_dir))
+$(xpt_files): $(process_py) $(call mkdir_deps,$(idl_deps_dir) $(dist_include_dir))
 
 $(call include_deps,$(depends_files))
 
 .PHONY: xpidl
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -1152,21 +1152,16 @@ endif
 endif
 
 ################################################################################
 # Install a linked .xpt into the appropriate place.
 # This should ideally be performed by the non-recursive idl make file. Some day.
 ifdef XPT_NAME #{
 
 ifndef NO_DIST_INSTALL
-_XPT_NAME_FILES := $(DEPTH)/config/makefiles/xpidl/xpt/$(XPT_NAME)
-_XPT_NAME_DEST := $(FINAL_TARGET)/components
-_XPT_NAME_TARGET := misc
-INSTALL_TARGETS += _XPT_NAME
-
 ifndef NO_INTERFACES_MANIFEST
 misc:: $(call mkdir_deps,$(FINAL_TARGET)/components)
 	$(call py_action,buildlist,$(FINAL_TARGET)/components/interfaces.manifest 'interfaces $(XPT_NAME)')
 	$(call py_action,buildlist,$(FINAL_TARGET)/chrome.manifest 'manifest components/interfaces.manifest')
 endif
 endif
 
 endif #} XPT_NAME
deleted file mode 100644
--- a/js/xpconnect/tests/idl/Makefile.in
+++ /dev/null
@@ -1,11 +0,0 @@
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-include $(topsrcdir)/config/rules.mk
-
-componentdir = js/xpconnect/tests/components
-libs:: $(DEPTH)/config/makefiles/xpidl/xpt/$(XPT_NAME)
-	$(INSTALL) $^ $(testxpcobjdir)/$(componentdir)/native
-	$(INSTALL) $^ $(testxpcobjdir)/$(componentdir)/js
--- a/js/xpconnect/tests/idl/moz.build
+++ b/js/xpconnect/tests/idl/moz.build
@@ -9,8 +9,16 @@ XPIDL_SOURCES += [
     'xpctest_bug809674.idl',
     'xpctest_interfaces.idl',
     'xpctest_params.idl',
     'xpctest_returncode.idl',
 ]
 
 XPIDL_MODULE = 'xpctest'
 
+# XXX: This relies on xpctest.xpt being created in dist/bin/components/ during
+# the export tier AND TEST_HARNESS_FILES being processed after that.
+TEST_HARNESS_FILES.xpcshell.js.xpconnect.tests.components.native += [
+    '!/dist/bin/components/xpctest.xpt',
+]
+TEST_HARNESS_FILES.xpcshell.js.xpconnect.tests.components.js += [
+    '!/dist/bin/components/xpctest.xpt',
+]
--- a/python/mozbuild/mozbuild/backend/common.py
+++ b/python/mozbuild/mozbuild/backend/common.py
@@ -34,17 +34,17 @@ class XPIDLManager(object):
     def __init__(self, config):
         self.config = config
         self.topsrcdir = config.topsrcdir
         self.topobjdir = config.topobjdir
 
         self.idls = {}
         self.modules = {}
 
-    def register_idl(self, source, module, allow_existing=False):
+    def register_idl(self, source, module, install_target, allow_existing=False):
         """Registers an IDL file with this instance.
 
         The IDL file will be built, installed, etc.
         """
         basename = mozpath.basename(source)
         root = mozpath.splitext(basename)[0]
 
         entry = {
@@ -53,17 +53,18 @@ class XPIDLManager(object):
             'basename': basename,
             'root': root,
         }
 
         if not allow_existing and entry['basename'] in self.idls:
             raise Exception('IDL already registered: %' % entry['basename'])
 
         self.idls[entry['basename']] = entry
-        self.modules.setdefault(entry['module'], set()).add(entry['root'])
+        t = self.modules.setdefault(entry['module'], (install_target, set()))
+        t[1].add(entry['root'])
 
 
 class WebIDLCollection(object):
     """Collects WebIDL info referenced during the build."""
 
     def __init__(self):
         self.sources = set()
         self.generated_sources = set()
@@ -176,17 +177,18 @@ class CommonBackend(BuildBackend):
         self._configs.add(obj.config)
 
         if isinstance(obj, TestManifest):
             for test in obj.tests:
                 self._test_manager.add(test, flavor=obj.flavor,
                     topsrcdir=obj.topsrcdir)
 
         elif isinstance(obj, XPIDLFile):
-            self._idl_manager.register_idl(obj.source_path, obj.module)
+            self._idl_manager.register_idl(obj.source_path, obj.module,
+                obj.install_target)
 
         elif isinstance(obj, ConfigFileSubstitution):
             # Do not handle ConfigFileSubstitution for Makefiles. Leave that
             # to other
             if mozpath.basename(obj.output_path) == 'Makefile':
                 return
             with self._get_preprocessor(obj) as pp:
                 pp.do_include(obj.input_path)
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -1,30 +1,31 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
+import errno
 import itertools
 import json
 import logging
 import os
 import re
 import types
 
 from collections import (
     defaultdict,
     namedtuple,
 )
+from StringIO import StringIO
 
 import mozwebidlcodegen
 from reftest import ReftestManifest
 
-import mozbuild.makeutil as mozmakeutil
 from mozpack.copier import FilePurger
 from mozpack.manifests import (
     InstallManifest,
 )
 import mozpack.path as mozpath
 
 from .common import CommonBackend
 from ..frontend.data import (
@@ -746,17 +747,17 @@ class RecursiveMakeBackend(CommonBackend
                             continue
                         if objdir == self.environment.topobjdir:
                             continue
                         self._no_skip['tools'].add(mozpath.relpath(objdir,
                             self.environment.topobjdir))
 
         # Write out a master list of all IPDL source files.
         ipdl_dir = mozpath.join(self.environment.topobjdir, 'ipc', 'ipdl')
-        mk = mozmakeutil.Makefile()
+        mk = Makefile()
 
         sorted_ipdl_sources = list(sorted(self._ipdl_sources))
         mk.add_statement('ALL_IPDLSRCS := %s' % ' '.join(sorted_ipdl_sources))
 
         def files_from(ipdl):
             base = mozpath.basename(ipdl)
             root, ext = mozpath.splitext(base)
 
@@ -978,72 +979,82 @@ INSTALL_TARGETS += %(prefix)s
         manifest = self._install_manifests['tests']
 
         for source, dest, _ in self._walk_hierarchy(obj, obj.modules):
             manifest.add_symlink(source, mozpath.join('modules', dest))
 
     def _handle_idl_manager(self, manager):
         build_files = self._install_manifests['xpidl']
 
-        for p in ('Makefile', 'backend.mk', '.deps/.mkdir.done',
-            'xpt/.mkdir.done'):
+        for p in ('Makefile', 'backend.mk', '.deps/.mkdir.done'):
             build_files.add_optional_exists(p)
 
         for idl in manager.idls.values():
             self._install_manifests['dist_idl'].add_symlink(idl['source'],
                 idl['basename'])
             self._install_manifests['dist_include'].add_optional_exists('%s.h'
                 % idl['root'])
 
         for module in manager.modules:
-            build_files.add_optional_exists(mozpath.join('xpt',
-                '%s.xpt' % module))
             build_files.add_optional_exists(mozpath.join('.deps',
                 '%s.pp' % module))
 
         modules = manager.modules
         xpt_modules = sorted(modules.keys())
-        rules = []
+        xpt_files = set()
+
+        mk = Makefile()
 
         for module in xpt_modules:
-            deps = sorted(modules[module])
-            idl_deps = ['$(dist_idl_dir)/%s.idl' % dep for dep in deps]
-            rules.extend([
-                # It may seem strange to have the .idl files listed as
-                # prerequisites both here and in the auto-generated .pp files.
-                # It is necessary to list them here to handle the case where a
-                # new .idl is added to an xpt. If we add a new .idl and nothing
-                # else has changed, the new .idl won't be referenced anywhere
-                # except in the command invocation. Therefore, the .xpt won't
-                # be rebuilt because the dependencies say it is up to date. By
-                # listing the .idls here, we ensure the make file has a
-                # reference to the new .idl. Since the new .idl presumably has
-                # an mtime newer than the .xpt, it will trigger xpt generation.
-                '$(idl_xpt_dir)/%s.xpt: %s' % (module, ' '.join(idl_deps)),
-                '\t@echo "$(notdir $@)"',
-                '\t$(idlprocess) $(basename $(notdir $@)) %s' % ' '.join(deps),
-                '',
-            ])
+            install_target, sources = modules[module]
+            deps = sorted(sources)
+
+            # It may seem strange to have the .idl files listed as
+            # prerequisites both here and in the auto-generated .pp files.
+            # It is necessary to list them here to handle the case where a
+            # new .idl is added to an xpt. If we add a new .idl and nothing
+            # else has changed, the new .idl won't be referenced anywhere
+            # except in the command invocation. Therefore, the .xpt won't
+            # be rebuilt because the dependencies say it is up to date. By
+            # listing the .idls here, we ensure the make file has a
+            # reference to the new .idl. Since the new .idl presumably has
+            # an mtime newer than the .xpt, it will trigger xpt generation.
+            xpt_path = '$(DEPTH)/%s/components/%s.xpt' % (install_target, module)
+            xpt_files.add(xpt_path)
+            rule = mk.create_rule([xpt_path])
+            rule.add_dependencies(['$(call mkdir_deps,%s)' % mozpath.dirname(xpt_path)])
+            rule.add_dependencies(manager.idls['%s.idl' % dep]['source'] for dep in deps)
+
+            if install_target.startswith('dist/'):
+                path = mozpath.relpath(xpt_path, '$(DEPTH)/dist')
+                prefix, subpath = path.split('/', 1)
+                key = 'dist_%s' % prefix
+
+                self._install_manifests[key].add_optional_exists(subpath)
+
+        rules = StringIO()
+        mk.dump(rules, removal_guard=False)
 
         # Create dependency for output header so we force regeneration if the
         # header was deleted. This ideally should not be necessary. However,
         # some processes (such as PGO at the time this was implemented) wipe
         # out dist/include without regard to our install manifests.
 
         obj = self.Substitution()
         obj.output_path = mozpath.join(self.environment.topobjdir, 'config',
             'makefiles', 'xpidl', 'Makefile')
         obj.input_path = mozpath.join(self.environment.topsrcdir, 'config',
             'makefiles', 'xpidl', 'Makefile.in')
         obj.topsrcdir = self.environment.topsrcdir
         obj.topobjdir = self.environment.topobjdir
         obj.config = self.environment
         self._create_makefile(obj, extra=dict(
-            xpidl_rules='\n'.join(rules),
+            xpidl_rules=rules.getvalue(),
             xpidl_modules=' '.join(xpt_modules),
+            xpt_files=' '.join(sorted(xpt_files)),
         ))
 
     def _process_program(self, program, backend_file):
         backend_file.write('PROGRAM = %s\n' % program)
 
     def _process_host_program(self, program, backend_file):
         backend_file.write('HOST_PROGRAM = %s\n' % program)
 
--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -1156,17 +1156,17 @@ VARIABLES = {
         which subdirectory they should be exported to. For example,
         to export ``foo.py`` to ``_tests/foo``, append to
         ``TEST_HARNESS_FILES`` like so::
            TEST_HARNESS_FILES.foo += ['foo.py']
 
         Files from topsrcdir and the objdir can also be installed by prefixing
         the path(s) with a '/' character and a '!' character, respectively::
            TEST_HARNESS_FILES.path += ['/build/bar.py', '!quux.py']
-        """, None),
+        """, 'libs'),
 }
 
 # Sanity check: we don't want any variable above to have a list as storage type.
 for name, (storage_type, input_types, docs, tier) in VARIABLES.items():
     if storage_type == list:
         raise RuntimeError('%s has a "list" storage type. Use "List" instead.'
             % name)
 
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -152,26 +152,29 @@ class VariablePassthru(ContextDerived):
         ContextDerived.__init__(self, context)
         self.variables = {}
 
 class XPIDLFile(ContextDerived):
     """Describes an XPIDL file to be compiled."""
 
     __slots__ = (
         'basename',
+        'install_target',
         'source_path',
     )
 
     def __init__(self, context, source, module):
         ContextDerived.__init__(self, context)
 
         self.source_path = source
         self.basename = mozpath.basename(source)
         self.module = module
 
+        self.install_target = context['FINAL_TARGET']
+
 class Defines(ContextDerived):
     """Context derived container object for DEFINES, which is an OrderedDict.
     """
     __slots__ = ('defines')
 
     def __init__(self, context, defines):
         ContextDerived.__init__(self, context)
         self.defines = defines
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -340,16 +340,24 @@ class TreeMetadataEmitter(LoggingMixin):
         else:
             return ExternalSharedLibrary(context, name)
 
     def emit_from_context(self, context):
         """Convert a Context to tree metadata objects.
 
         This is a generator of mozbuild.frontend.data.ContextDerived instances.
         """
+
+        # We only want to emit an InstallationTarget if one of the consulted
+        # variables is defined. Later on, we look up FINAL_TARGET, which has
+        # the side-effect of populating it. So, we need to do this lookup
+        # early.
+        if any(k in context for k in ('FINAL_TARGET', 'XPI_NAME', 'DIST_SUBDIR')):
+            yield InstallationTarget(context)
+
         # We always emit a directory traversal descriptor. This is needed by
         # the recursive make backend.
         for o in self._emit_directory_traversal_from_context(context): yield o
 
         for path in context['CONFIGURE_SUBST_FILES']:
             yield self._create_substitution(ConfigFileSubstitution, context,
                 path)
 
@@ -518,19 +526,19 @@ class TreeMetadataEmitter(LoggingMixin):
             for path, strings in test_harness_files.walk():
                 if not path and strings:
                     raise SandboxValidationError(
                         'Cannot install files to the root of TEST_HARNESS_FILES', context)
 
                 for s in strings:
                     if context.is_objdir_path(s):
                         if s.startswith('!/'):
-                            raise SandboxValidationError(
-                                'Topobjdir-relative file not allowed in TEST_HARNESS_FILES: %s' % s, context)
-                        objdir_files[path].append(s[1:])
+                            objdir_files[path].append('$(DEPTH)/%s' % s[2:])
+                        else:
+                            objdir_files[path].append(s[1:])
                     else:
                         resolved = context.resolve_path(s)
                         if '*' in s:
                             srcdir_pattern_files[path].append(s);
                         elif not os.path.exists(resolved):
                             raise SandboxValidationError(
                                 'File listed in TEST_HARNESS_FILES does not exist: %s' % s, context)
                         else:
@@ -611,20 +619,16 @@ class TreeMetadataEmitter(LoggingMixin):
                 relative_include = local_include
 
             actual_include = os.path.join(path, relative_include)
             if not os.path.exists(actual_include):
                 raise SandboxValidationError('Path specified in LOCAL_INCLUDES '
                     'does not exist: %s (resolved to %s)' % (local_include, actual_include), context)
             yield LocalInclude(context, local_include)
 
-        if context.get('FINAL_TARGET') or context.get('XPI_NAME') or \
-                context.get('DIST_SUBDIR'):
-            yield InstallationTarget(context)
-
         final_target_files = context.get('FINAL_TARGET_FILES')
         if final_target_files:
             yield FinalTargetFiles(context, final_target_files, context['FINAL_TARGET'])
 
         host_libname = context.get('HOST_LIBRARY_NAME')
         libname = context.get('LIBRARY_NAME')
 
         if host_libname:
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -438,17 +438,19 @@ class TestRecursiveMakeBackend(BackendTe
 
         m = InstallManifest(path=mozpath.join(install_dir, 'dist_idl'))
         self.assertEqual(len(m), 2)
         self.assertIn('bar.idl', m)
         self.assertIn('foo.idl', m)
 
         m = InstallManifest(path=mozpath.join(install_dir, 'xpidl'))
         self.assertIn('.deps/my_module.pp', m)
-        self.assertIn('xpt/my_module.xpt', m)
+
+        m = InstallManifest(path=os.path.join(install_dir, 'dist_bin'))
+        self.assertIn('components/my_module.xpt', m)
 
         m = InstallManifest(path=mozpath.join(install_dir, 'dist_include'))
         self.assertIn('foo.h', m)
 
         p = mozpath.join(env.topobjdir, 'config/makefiles/xpidl')
         self.assertTrue(os.path.isdir(p))
 
         self.assertTrue(os.path.isfile(mozpath.join(p, 'Makefile')))