Bug 896797 - Part 3: Use install manifests for managing dist/include; r=glandium
☠☠ backed out by 1faf7273eaae ☠ ☠
authorGregory Szorc <gps@mozilla.com>
Tue, 03 Sep 2013 22:51:58 -0700
changeset 158331 8090e534656ae573a7623ef6afee8fa2b3ec51c1
parent 158330 3cf5058f6296272089460ffd710173a8d05abae4
child 158332 1791f7250a6c2617c1cc30632cdc42f52b358c55
push id1
push usersledru@mozilla.com
push dateThu, 04 Dec 2014 17:57:20 +0000
reviewersglandium
bugs896797
milestone26.0a1
Bug 896797 - Part 3: Use install manifests for managing dist/include; r=glandium
Makefile.in
js/src/Makefile.in
python/mozbuild/mozbuild/action/process_install_manifest.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -57,16 +57,17 @@ ifndef MOZ_PROFILE_USE
 # necessary. To manage new directories or add files to the manifests,
 # modify the backend generator.
 #
 # We need to explicitly put backend.RecursiveMakeBackend.built here
 # otherwise the rule in rules.mk doesn't run early enough.
 default alldep all:: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built
 	$(call SUBMAKE,backend.RecursiveMakeBackend.built,js/src,1)
 	$(call py_action,purge_manifests,-d _build_manifests/purge .)
+	$(call py_action,process_install_manifest,$(DIST)/include _build_manifests/install/dist_include js/src/_build_manifests/install/dist_include)
 endif
 
 CLOBBER: $(topsrcdir)/CLOBBER
 	@echo "STOP!  The CLOBBER file has changed."
 	@echo "Please run the build through a sanctioned build wrapper, such as"
 	@echo "'mach build' or client.mk."
 	@exit 1
 
--- a/js/src/Makefile.in
+++ b/js/src/Makefile.in
@@ -172,16 +172,17 @@ USE_HOST_CXX = 1
 ifdef HAVE_DTRACE
 ifneq ($(OS_ARCH),Darwin)
 DTRACE_PROBE_OBJ = $(LIBRARY_NAME)-dtrace.$(OBJ_SUFFIX)
 endif
 MOZILLA_DTRACE_SRC = $(srcdir)/devtools/javascript-trace.d
 endif
 
 default::
+	$(call py_action,process_install_manifest,--no-remove $(DIST)/include _build_manifests/install/dist_include)
 
 ifneq (,$(CROSS_COMPILE)$(filter-out WINNT OS2,$(OS_ARCH)))
 # nsinstall doesn't get built until we enter config/ in the exports phase,
 # so we'll have to manually ensure it gets built here if we want to use
 # $(EXPORTS)
 export:: config/nsinstall$(HOST_BIN_SUFFIX)
 $(PUBLIC) $(SDK_PUBLIC): config/nsinstall$(HOST_BIN_SUFFIX)
 
--- a/python/mozbuild/mozbuild/action/process_install_manifest.py
+++ b/python/mozbuild/mozbuild/action/process_install_manifest.py
@@ -8,34 +8,37 @@ import argparse
 from mozpack.copier import FileCopier
 from mozpack.manifests import InstallManifest
 
 
 COMPLETE = 'From {dest}: Kept {existing} existing; Added/updated {updated}; ' \
     'Removed {rm_files} files and {rm_dirs} directories.'
 
 
-def process_manifest(destdir, *paths):
+def process_manifest(destdir, paths, remove_unaccounted=True):
     manifest = InstallManifest()
     for path in paths:
         manifest |= InstallManifest(path=path)
 
     copier = FileCopier()
     manifest.populate_registry(copier)
-    return copier.copy(destdir)
+    return copier.copy(destdir, remove_unaccounted=remove_unaccounted)
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(
         description='Process install manifest files.')
 
     parser.add_argument('destdir', help='Destination directory.')
     parser.add_argument('manifests', nargs='+', help='Path to manifest file(s).')
+    parser.add_argument('--no-remove', action='store_true',
+        help='Do not remove unaccounted files from destination.')
 
     args = parser.parse_args()
 
-    result = process_manifest(args.destdir, *args.manifests)
+    result = process_manifest(args.destdir, args.manifests,
+        remove_unaccounted=not args.no_remove)
 
     print(COMPLETE.format(dest=args.destdir,
         existing=result.existing_files_count,
         updated=result.updated_files_count,
         rm_files=result.removed_files_count,
         rm_dirs=result.removed_directories_count))
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -144,30 +144,28 @@ class RecursiveMakeBackend(CommonBackend
         self.summary.backend_detailed_summary = types.MethodType(detailed,
             self.summary)
 
         self.xpcshell_manifests = []
 
         self.backend_input_files.add(os.path.join(self.environment.topobjdir,
             'config', 'autoconf.mk'))
 
-        self._install_manifests = dict()
-
         self._purge_manifests = dict(
             dist_bin=PurgeManifest(relpath='dist/bin'),
-            dist_include=PurgeManifest(relpath='dist/include'),
             dist_private=PurgeManifest(relpath='dist/private'),
             dist_public=PurgeManifest(relpath='dist/public'),
             dist_sdk=PurgeManifest(relpath='dist/sdk'),
             tests=PurgeManifest(relpath='_tests'),
             xpidl=PurgeManifest(relpath='config/makefiles/xpidl'),
         )
 
         self._install_manifests = dict(
             dist_idl=InstallManifest(),
+            dist_include=InstallManifest(),
         )
 
     def _update_from_avoid_write(self, result):
         existed, updated = result
 
         if not existed:
             self.summary.created_count += 1
         elif updated:
@@ -363,54 +361,50 @@ class RecursiveMakeBackend(CommonBackend
         if len(obj.external_make_dirs):
             fh.write('DIRS += %s\n' % ' '.join(obj.external_make_dirs))
 
         if len(obj.parallel_external_make_dirs):
             fh.write('PARALLEL_DIRS += %s\n' %
                 ' '.join(obj.parallel_external_make_dirs))
 
     def _process_exports(self, obj, exports, backend_file, namespace=""):
+        # This may not be needed, but is present for backwards compatibility
+        # with the old make rules, just in case.
+        if not obj.dist_install:
+            return
+
         strings = exports.get_strings()
         if namespace:
-            if strings:
-                backend_file.write('EXPORTS_NAMESPACES += %s\n' % namespace)
-            export_name = 'EXPORTS_%s' % namespace
             namespace += '/'
-        else:
-            export_name = 'EXPORTS'
 
-        # Iterate over the list of export filenames, printing out an EXPORTS
-        # declaration for each.
-        if strings:
-            backend_file.write('%s += %s\n' % (export_name, ' '.join(strings)))
+        for s in strings:
+            source = os.path.normpath(os.path.join(obj.srcdir, s))
+            dest = '%s%s' % (namespace, os.path.basename(s))
+            self._install_manifests['dist_include'].add_symlink(source, dest)
 
-            for s in strings:
-                source = os.path.normpath(os.path.join(obj.srcdir, s))
-                if not os.path.isfile(source):
-                    raise Exception('File listed in EXPORTS does not exist: %s' % source)
-
-                p = '%s%s' % (namespace, s)
-                self._purge_manifests['dist_include'].add(p)
+            if not os.path.exists(source):
+                raise Exception('File listed in EXPORTS does not exist: %s' % source)
 
         children = exports.get_children()
         for subdir in sorted(children):
             self._process_exports(obj, children[subdir], backend_file,
                 namespace=namespace + subdir)
 
     def _handle_idl_manager(self, manager):
         build_files = self._purge_manifests['xpidl']
 
         for p in ('Makefile', 'backend.mk', '.deps/.mkdir.done',
             'headers/.mkdir.done', 'xpt/.mkdir.done'):
             build_files.add(p)
 
         for idl in manager.idls.values():
             self._install_manifests['dist_idl'].add_symlink(idl['source'],
                 idl['basename'])
-            self._purge_manifests['dist_include'].add('%s.h' % idl['root'])
+            self._install_manifests['dist_include'].add_optional_exists('%s.h'
+                % idl['root'])
             build_files.add(mozpath.join('headers', '%s.h' % idl['root']))
 
         for module in manager.modules:
             build_files.add(mozpath.join('xpt', '%s.xpt' % module))
             build_files.add(mozpath.join('.deps', '%s.pp' % module))
 
         headers = sorted('%s.h' % idl['root'] for idl in manager.idls.values())
         modules = manager.modules
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -156,21 +156,23 @@ class XPIDLFile(SandboxDerived):
 
 class Exports(SandboxDerived):
     """Sandbox container object for EXPORTS, which is a HierarchicalStringList.
 
     We need an object derived from SandboxDerived for use in the backend, so
     this object fills that role. It just has a reference to the underlying
     HierarchicalStringList, which is created when parsing EXPORTS.
     """
-    __slots__ = ('exports')
+    __slots__ = ('exports', 'dist_install')
 
-    def __init__(self, sandbox, exports):
+    def __init__(self, sandbox, exports, dist_install=True):
         SandboxDerived.__init__(self, sandbox)
         self.exports = exports
+        self.dist_install = dist_install
+
 
 class IPDLFile(SandboxDerived):
     """Describes an individual .ipdl source file."""
 
     __slots__ = (
         'basename',
     )
 
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -149,17 +149,18 @@ class TreeMetadataEmitter(LoggingMixin):
             if sandbox[moz]:
                 passthru.variables[mak] = sandbox[moz]
 
         if passthru.variables:
             yield passthru
 
         exports = sandbox.get('EXPORTS')
         if exports:
-            yield Exports(sandbox, exports)
+            yield Exports(sandbox, exports,
+                dist_install=not sandbox.get('NO_DIST_INSTALL', False))
 
         program = sandbox.get('PROGRAM')
         if program:
             yield Program(sandbox, program, sandbox['CONFIG']['BIN_SUFFIX'])
 
         for manifest in sandbox.get('XPCSHELL_TESTS_MANIFESTS', []):
             yield XpcshellManifests(sandbox, manifest)
 
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -229,43 +229,26 @@ class TestRecursiveMakeBackend(BackendTe
         }
 
         for var, val in expected.items():
             # print("test_variable_passthru[%s]" % (var))
             found = [str for str in lines if str.startswith(var)]
             self.assertEqual(found, val)
 
     def test_exports(self):
-        """Ensure EXPORTS is written out correctly."""
+        """Ensure EXPORTS is handled properly."""
         env = self._consume('exports', RecursiveMakeBackend)
 
-        backend_path = os.path.join(env.topobjdir, 'backend.mk')
-        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
-
-        self.assertEqual(lines, [
-            'MOZBUILD_DERIVED := 1',
-            'NO_MAKEFILE_RULE := 1',
-            'NO_SUBMAKEFILES_RULE := 1',
-            'EXPORTS += foo.h',
-            'EXPORTS_NAMESPACES += mozilla',
-            'EXPORTS_mozilla += mozilla1.h mozilla2.h',
-            'EXPORTS_NAMESPACES += mozilla/dom',
-            'EXPORTS_mozilla/dom += dom1.h dom2.h',
-            'EXPORTS_NAMESPACES += mozilla/gfx',
-            'EXPORTS_mozilla/gfx += gfx.h',
-            'EXPORTS_NAMESPACES += nspr/private',
-            'EXPORTS_nspr/private += pprio.h',
-        ])
-
-        # EXPORTS files should appear in the dist_include purge manifest.
-        m = PurgeManifest(path=os.path.join(env.topobjdir,
-            '_build_manifests', 'purge', 'dist_include'))
-        self.assertIn('foo.h', m.entries)
-        self.assertIn('mozilla/mozilla1.h', m.entries)
-        self.assertIn('mozilla/dom/dom2.h', m.entries)
+        # EXPORTS files should appear in the dist_include install manifest.
+        m = InstallManifest(path=os.path.join(env.topobjdir,
+            '_build_manifests', 'install', 'dist_include'))
+        self.assertEqual(len(m), 7)
+        self.assertIn('foo.h', m)
+        self.assertIn('mozilla/mozilla1.h', m)
+        self.assertIn('mozilla/dom/dom2.h', m)
 
     def test_xpcshell_manifests(self):
         """Ensure XPCSHELL_TESTS_MANIFESTS is written out correctly."""
         env = self._consume('xpcshell_manifests', RecursiveMakeBackend)
 
         backend_path = os.path.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
@@ -292,18 +275,18 @@ class TestRecursiveMakeBackend(BackendTe
         self.assertIn('.deps/my_module.pp', m.entries)
         self.assertIn('xpt/my_module.xpt', m.entries)
 
         m = InstallManifest(path=os.path.join(install_dir, 'dist_idl'))
         self.assertEqual(len(m), 2)
         self.assertIn('bar.idl', m)
         self.assertIn('foo.idl', m)
 
-        m = PurgeManifest(path=os.path.join(purge_dir, 'dist_include'))
-        self.assertIn('foo.h', m.entries)
+        m = InstallManifest(path=os.path.join(install_dir, 'dist_include'))
+        self.assertIn('foo.h', m)
 
         p = os.path.join(env.topobjdir, 'config/makefiles/xpidl')
         self.assertTrue(os.path.isdir(p))
 
         self.assertTrue(os.path.isfile(os.path.join(p, 'Makefile')))
 
     def test_xpcshell_master_manifest(self):
         """Ensure that the master xpcshell manifest is written out correctly."""
@@ -320,17 +303,16 @@ class TestRecursiveMakeBackend(BackendTe
     def test_purge_manifests_written(self):
         env = self._consume('stub0', RecursiveMakeBackend)
 
         purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
         self.assertTrue(os.path.exists(purge_dir))
 
         expected = [
             'dist_bin',
-            'dist_include',
             'dist_private',
             'dist_public',
             'dist_sdk',
             'tests',
         ]
 
         for e in expected:
             full = os.path.join(purge_dir, e)