Bug 896797 - Part 3: Use install manifests for managing dist/include; r=glandium
authorGregory Szorc <gps@mozilla.com>
Tue, 03 Sep 2013 22:51:58 -0700
changeset 158461 743278b7340f88d6d5071bbd1b4fd79bf35c21f8
parent 158460 a895979d4c8ba4aaa51f6be40e37df076b6646a2
child 158462 0343a9d0f208a20d97d2eb2b40bc54df870e679a
push id2961
push userlsblakk@mozilla.com
push dateMon, 28 Oct 2013 21:59:28 +0000
treeherdermozilla-beta@73ef4f13486f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs896797
milestone26.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 896797 - Part 3: Use install manifests for managing dist/include; r=glandium
Makefile.in
js/src/Makefile.in
python/mozbuild/mozbuild/action/process_install_manifest.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -94,16 +94,17 @@ include $(topsrcdir)/testing/testsuite-t
 endif
 
 # Hacky way for precompile tier to bypass default tier traversal mechanism.
 TIER_precompile_CUSTOM := 1
 
 include $(topsrcdir)/config/rules.mk
 
 default all alldep::
+	$(call py_action,process_install_manifest,$(DIST)/include _build_manifests/install/dist_include js/src/_build_manifests/install/dist_include)
 	$(call BUILDSTATUS,TIERS $(TIERS))
 	$(foreach tier,$(TIERS),$(call SUBMAKE,tier_$(tier)))
 
 include $(topsrcdir)/config/makefiles/tiers.mk
 $(foreach tier,$(TIERS),$(eval $(call CREATE_TIER_RULE,$(tier))))
 
 distclean::
 	$(RM) $(DIST_GARBAGE)
--- a/js/src/Makefile.in
+++ b/js/src/Makefile.in
@@ -172,16 +172,17 @@ USE_HOST_CXX = 1
 ifdef HAVE_DTRACE
 ifneq ($(OS_ARCH),Darwin)
 DTRACE_PROBE_OBJ = $(LIBRARY_NAME)-dtrace.$(OBJ_SUFFIX)
 endif
 MOZILLA_DTRACE_SRC = $(srcdir)/devtools/javascript-trace.d
 endif
 
 default::
+	$(call py_action,process_install_manifest,--no-remove $(DIST)/include _build_manifests/install/dist_include)
 
 ifneq (,$(CROSS_COMPILE)$(filter-out WINNT OS2,$(OS_ARCH)))
 # nsinstall doesn't get built until we enter config/ in the exports phase,
 # so we'll have to manually ensure it gets built here if we want to use
 # $(EXPORTS)
 export:: config/nsinstall$(HOST_BIN_SUFFIX)
 $(PUBLIC) $(SDK_PUBLIC): config/nsinstall$(HOST_BIN_SUFFIX)
 
--- a/python/mozbuild/mozbuild/action/process_install_manifest.py
+++ b/python/mozbuild/mozbuild/action/process_install_manifest.py
@@ -8,34 +8,37 @@ import argparse
 from mozpack.copier import FileCopier
 from mozpack.manifests import InstallManifest
 
 
 COMPLETE = 'From {dest}: Kept {existing} existing; Added/updated {updated}; ' \
     'Removed {rm_files} files and {rm_dirs} directories.'
 
 
-def process_manifest(destdir, *paths):
+def process_manifest(destdir, paths, remove_unaccounted=True):
     manifest = InstallManifest()
     for path in paths:
         manifest |= InstallManifest(path=path)
 
     copier = FileCopier()
     manifest.populate_registry(copier)
-    return copier.copy(destdir)
+    return copier.copy(destdir, remove_unaccounted=remove_unaccounted)
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(
         description='Process install manifest files.')
 
     parser.add_argument('destdir', help='Destination directory.')
     parser.add_argument('manifests', nargs='+', help='Path to manifest file(s).')
+    parser.add_argument('--no-remove', action='store_true',
+        help='Do not remove unaccounted files from destination.')
 
     args = parser.parse_args()
 
-    result = process_manifest(args.destdir, *args.manifests)
+    result = process_manifest(args.destdir, args.manifests,
+        remove_unaccounted=not args.no_remove)
 
     print(COMPLETE.format(dest=args.destdir,
         existing=result.existing_files_count,
         updated=result.updated_files_count,
         rm_files=result.removed_files_count,
         rm_dirs=result.removed_directories_count))
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -144,30 +144,28 @@ class RecursiveMakeBackend(CommonBackend
         self.summary.backend_detailed_summary = types.MethodType(detailed,
             self.summary)
 
         self.xpcshell_manifests = []
 
         self.backend_input_files.add(os.path.join(self.environment.topobjdir,
             'config', 'autoconf.mk'))
 
-        self._install_manifests = dict()
-
         self._purge_manifests = dict(
             dist_bin=PurgeManifest(relpath='dist/bin'),
-            dist_include=PurgeManifest(relpath='dist/include'),
             dist_private=PurgeManifest(relpath='dist/private'),
             dist_public=PurgeManifest(relpath='dist/public'),
             dist_sdk=PurgeManifest(relpath='dist/sdk'),
             tests=PurgeManifest(relpath='_tests'),
             xpidl=PurgeManifest(relpath='config/makefiles/xpidl'),
         )
 
         self._install_manifests = dict(
             dist_idl=InstallManifest(),
+            dist_include=InstallManifest(),
         )
 
     def _update_from_avoid_write(self, result):
         existed, updated = result
 
         if not existed:
             self.summary.created_count += 1
         elif updated:
@@ -363,54 +361,50 @@ class RecursiveMakeBackend(CommonBackend
         if len(obj.external_make_dirs):
             fh.write('DIRS += %s\n' % ' '.join(obj.external_make_dirs))
 
         if len(obj.parallel_external_make_dirs):
             fh.write('PARALLEL_DIRS += %s\n' %
                 ' '.join(obj.parallel_external_make_dirs))
 
     def _process_exports(self, obj, exports, backend_file, namespace=""):
+        # This may not be needed, but is present for backwards compatibility
+        # with the old make rules, just in case.
+        if not obj.dist_install:
+            return
+
         strings = exports.get_strings()
         if namespace:
-            if strings:
-                backend_file.write('EXPORTS_NAMESPACES += %s\n' % namespace)
-            export_name = 'EXPORTS_%s' % namespace
             namespace += '/'
-        else:
-            export_name = 'EXPORTS'
 
-        # Iterate over the list of export filenames, printing out an EXPORTS
-        # declaration for each.
-        if strings:
-            backend_file.write('%s += %s\n' % (export_name, ' '.join(strings)))
+        for s in strings:
+            source = os.path.normpath(os.path.join(obj.srcdir, s))
+            dest = '%s%s' % (namespace, os.path.basename(s))
+            self._install_manifests['dist_include'].add_symlink(source, dest)
 
-            for s in strings:
-                source = os.path.normpath(os.path.join(obj.srcdir, s))
-                if not os.path.isfile(source):
-                    raise Exception('File listed in EXPORTS does not exist: %s' % source)
-
-                p = '%s%s' % (namespace, s)
-                self._purge_manifests['dist_include'].add(p)
+            if not os.path.exists(source):
+                raise Exception('File listed in EXPORTS does not exist: %s' % source)
 
         children = exports.get_children()
         for subdir in sorted(children):
             self._process_exports(obj, children[subdir], backend_file,
                 namespace=namespace + subdir)
 
     def _handle_idl_manager(self, manager):
         build_files = self._purge_manifests['xpidl']
 
         for p in ('Makefile', 'backend.mk', '.deps/.mkdir.done',
             'headers/.mkdir.done', 'xpt/.mkdir.done'):
             build_files.add(p)
 
         for idl in manager.idls.values():
             self._install_manifests['dist_idl'].add_symlink(idl['source'],
                 idl['basename'])
-            self._purge_manifests['dist_include'].add('%s.h' % idl['root'])
+            self._install_manifests['dist_include'].add_optional_exists('%s.h'
+                % idl['root'])
             build_files.add(mozpath.join('headers', '%s.h' % idl['root']))
 
         for module in manager.modules:
             build_files.add(mozpath.join('xpt', '%s.xpt' % module))
             build_files.add(mozpath.join('.deps', '%s.pp' % module))
 
         headers = sorted('%s.h' % idl['root'] for idl in manager.idls.values())
         modules = manager.modules
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -156,21 +156,23 @@ class XPIDLFile(SandboxDerived):
 
 class Exports(SandboxDerived):
     """Sandbox container object for EXPORTS, which is a HierarchicalStringList.
 
     We need an object derived from SandboxDerived for use in the backend, so
     this object fills that role. It just has a reference to the underlying
     HierarchicalStringList, which is created when parsing EXPORTS.
     """
-    __slots__ = ('exports')
+    __slots__ = ('exports', 'dist_install')
 
-    def __init__(self, sandbox, exports):
+    def __init__(self, sandbox, exports, dist_install=True):
         SandboxDerived.__init__(self, sandbox)
         self.exports = exports
+        self.dist_install = dist_install
+
 
 class IPDLFile(SandboxDerived):
     """Describes an individual .ipdl source file."""
 
     __slots__ = (
         'basename',
     )
 
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -149,17 +149,18 @@ class TreeMetadataEmitter(LoggingMixin):
             if sandbox[moz]:
                 passthru.variables[mak] = sandbox[moz]
 
         if passthru.variables:
             yield passthru
 
         exports = sandbox.get('EXPORTS')
         if exports:
-            yield Exports(sandbox, exports)
+            yield Exports(sandbox, exports,
+                dist_install=not sandbox.get('NO_DIST_INSTALL', False))
 
         program = sandbox.get('PROGRAM')
         if program:
             yield Program(sandbox, program, sandbox['CONFIG']['BIN_SUFFIX'])
 
         for manifest in sandbox.get('XPCSHELL_TESTS_MANIFESTS', []):
             yield XpcshellManifests(sandbox, manifest)
 
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -229,43 +229,26 @@ class TestRecursiveMakeBackend(BackendTe
         }
 
         for var, val in expected.items():
             # print("test_variable_passthru[%s]" % (var))
             found = [str for str in lines if str.startswith(var)]
             self.assertEqual(found, val)
 
     def test_exports(self):
-        """Ensure EXPORTS is written out correctly."""
+        """Ensure EXPORTS is handled properly."""
         env = self._consume('exports', RecursiveMakeBackend)
 
-        backend_path = os.path.join(env.topobjdir, 'backend.mk')
-        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
-
-        self.assertEqual(lines, [
-            'MOZBUILD_DERIVED := 1',
-            'NO_MAKEFILE_RULE := 1',
-            'NO_SUBMAKEFILES_RULE := 1',
-            'EXPORTS += foo.h',
-            'EXPORTS_NAMESPACES += mozilla',
-            'EXPORTS_mozilla += mozilla1.h mozilla2.h',
-            'EXPORTS_NAMESPACES += mozilla/dom',
-            'EXPORTS_mozilla/dom += dom1.h dom2.h',
-            'EXPORTS_NAMESPACES += mozilla/gfx',
-            'EXPORTS_mozilla/gfx += gfx.h',
-            'EXPORTS_NAMESPACES += nspr/private',
-            'EXPORTS_nspr/private += pprio.h',
-        ])
-
-        # EXPORTS files should appear in the dist_include purge manifest.
-        m = PurgeManifest(path=os.path.join(env.topobjdir,
-            '_build_manifests', 'purge', 'dist_include'))
-        self.assertIn('foo.h', m.entries)
-        self.assertIn('mozilla/mozilla1.h', m.entries)
-        self.assertIn('mozilla/dom/dom2.h', m.entries)
+        # EXPORTS files should appear in the dist_include install manifest.
+        m = InstallManifest(path=os.path.join(env.topobjdir,
+            '_build_manifests', 'install', 'dist_include'))
+        self.assertEqual(len(m), 7)
+        self.assertIn('foo.h', m)
+        self.assertIn('mozilla/mozilla1.h', m)
+        self.assertIn('mozilla/dom/dom2.h', m)
 
     def test_xpcshell_manifests(self):
         """Ensure XPCSHELL_TESTS_MANIFESTS is written out correctly."""
         env = self._consume('xpcshell_manifests', RecursiveMakeBackend)
 
         backend_path = os.path.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
@@ -292,18 +275,18 @@ class TestRecursiveMakeBackend(BackendTe
         self.assertIn('.deps/my_module.pp', m.entries)
         self.assertIn('xpt/my_module.xpt', m.entries)
 
         m = InstallManifest(path=os.path.join(install_dir, 'dist_idl'))
         self.assertEqual(len(m), 2)
         self.assertIn('bar.idl', m)
         self.assertIn('foo.idl', m)
 
-        m = PurgeManifest(path=os.path.join(purge_dir, 'dist_include'))
-        self.assertIn('foo.h', m.entries)
+        m = InstallManifest(path=os.path.join(install_dir, 'dist_include'))
+        self.assertIn('foo.h', m)
 
         p = os.path.join(env.topobjdir, 'config/makefiles/xpidl')
         self.assertTrue(os.path.isdir(p))
 
         self.assertTrue(os.path.isfile(os.path.join(p, 'Makefile')))
 
     def test_xpcshell_master_manifest(self):
         """Ensure that the master xpcshell manifest is written out correctly."""
@@ -320,17 +303,16 @@ class TestRecursiveMakeBackend(BackendTe
     def test_purge_manifests_written(self):
         env = self._consume('stub0', RecursiveMakeBackend)
 
         purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
         self.assertTrue(os.path.exists(purge_dir))
 
         expected = [
             'dist_bin',
-            'dist_include',
             'dist_private',
             'dist_public',
             'dist_sdk',
             'tests',
         ]
 
         for e in expected:
             full = os.path.join(purge_dir, e)