Bug 896797 - Part 3: Use install manifests for managing dist/include; r=glandium
☠☠ backed out by 28c308fbc854 ☠ ☠
authorGregory Szorc <gps@mozilla.com>
Tue, 03 Sep 2013 22:51:58 -0700
changeset 158325 ce03cc2994aa65f52659354c2816c14cb1ec6ac4
parent 158324 ae9f95fc113658570613ede2924602f81d38ed69
child 158326 b23aae5cb0c2a110006db5fbed53e0099464ec4d
push id2961
push userlsblakk@mozilla.com
push dateMon, 28 Oct 2013 21:59:28 +0000
treeherdermozilla-beta@73ef4f13486f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs896797
milestone26.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 896797 - Part 3: Use install manifests for managing dist/include; r=glandium
Makefile.in
js/src/Makefile.in
python/mozbuild/mozbuild/action/process_install_manifest.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -57,16 +57,17 @@ ifndef MOZ_PROFILE_USE
 # necessary. To manage new directories or add files to the manifests,
 # modify the backend generator.
 #
 # We need to explicitly put backend.RecursiveMakeBackend.built here
 # otherwise the rule in rules.mk doesn't run early enough.
 default alldep all:: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built
 	$(call SUBMAKE,backend.RecursiveMakeBackend.built,js/src,1)
 	$(call py_action,purge_manifests,-d _build_manifests/purge .)
+	$(call py_action,process_install_manifest,$(DIST)/include _build_manifests/install/dist_include js/src/_build_manifests/install/dist_include)
 endif
 
 CLOBBER: $(topsrcdir)/CLOBBER
 	@echo "STOP!  The CLOBBER file has changed."
 	@echo "Please run the build through a sanctioned build wrapper, such as"
 	@echo "'mach build' or client.mk."
 	@exit 1
 
--- a/js/src/Makefile.in
+++ b/js/src/Makefile.in
@@ -172,16 +172,17 @@ USE_HOST_CXX = 1
 ifdef HAVE_DTRACE
 ifneq ($(OS_ARCH),Darwin)
 DTRACE_PROBE_OBJ = $(LIBRARY_NAME)-dtrace.$(OBJ_SUFFIX)
 endif
 MOZILLA_DTRACE_SRC = $(srcdir)/devtools/javascript-trace.d
 endif
 
 default::
+	$(call py_action,process_install_manifest,$(if $(JS_STANDALONE),,--no-remove) $(DIST)/include _build_manifests/install/dist_include)
 
 ifneq (,$(CROSS_COMPILE)$(filter-out WINNT OS2,$(OS_ARCH)))
 # nsinstall doesn't get built until we enter config/ in the exports phase,
 # so we'll have to manually ensure it gets built here if we want to use
 # $(EXPORTS)
 export:: config/nsinstall$(HOST_BIN_SUFFIX)
 $(PUBLIC) $(SDK_PUBLIC): config/nsinstall$(HOST_BIN_SUFFIX)
 
--- a/python/mozbuild/mozbuild/action/process_install_manifest.py
+++ b/python/mozbuild/mozbuild/action/process_install_manifest.py
@@ -8,34 +8,37 @@ import argparse
 from mozpack.copier import FileCopier
 from mozpack.manifests import InstallManifest
 
 
 COMPLETE = 'From {dest}: Kept {existing} existing; Added/updated {updated}; ' \
     'Removed {rm_files} files and {rm_dirs} directories.'
 
 
-def process_manifest(destdir, *paths):
+def process_manifest(destdir, paths, remove_unaccounted=True):
     manifest = InstallManifest()
     for path in paths:
         manifest |= InstallManifest(path=path)
 
     copier = FileCopier()
     manifest.populate_registry(copier)
-    return copier.copy(destdir)
+    return copier.copy(destdir, remove_unaccounted=remove_unaccounted)
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(
         description='Process install manifest files.')
 
     parser.add_argument('destdir', help='Destination directory.')
     parser.add_argument('manifests', nargs='+', help='Path to manifest file(s).')
+    parser.add_argument('--no-remove', action='store_true',
+        help='Do not remove unaccounted files from destination.')
 
     args = parser.parse_args()
 
-    result = process_manifest(args.destdir, *args.manifests)
+    result = process_manifest(args.destdir, args.manifests,
+        remove_unaccounted=not args.no_remove)
 
     print(COMPLETE.format(dest=args.destdir,
         existing=result.existing_files_count,
         updated=result.updated_files_count,
         rm_files=result.removed_files_count,
         rm_dirs=result.removed_directories_count))
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -144,30 +144,28 @@ class RecursiveMakeBackend(CommonBackend
         self.summary.backend_detailed_summary = types.MethodType(detailed,
             self.summary)
 
         self.xpcshell_manifests = []
 
         self.backend_input_files.add(os.path.join(self.environment.topobjdir,
             'config', 'autoconf.mk'))
 
-        self._install_manifests = dict()
-
         self._purge_manifests = dict(
             dist_bin=PurgeManifest(relpath='dist/bin'),
-            dist_include=PurgeManifest(relpath='dist/include'),
             dist_private=PurgeManifest(relpath='dist/private'),
             dist_public=PurgeManifest(relpath='dist/public'),
             dist_sdk=PurgeManifest(relpath='dist/sdk'),
             tests=PurgeManifest(relpath='_tests'),
             xpidl=PurgeManifest(relpath='config/makefiles/xpidl'),
         )
 
         self._install_manifests = dict(
             dist_idl=InstallManifest(),
+            dist_include=InstallManifest(),
         )
 
     def _update_from_avoid_write(self, result):
         existed, updated = result
 
         if not existed:
             self.summary.created_count += 1
         elif updated:
@@ -363,54 +361,50 @@ class RecursiveMakeBackend(CommonBackend
         if len(obj.external_make_dirs):
             fh.write('DIRS += %s\n' % ' '.join(obj.external_make_dirs))
 
         if len(obj.parallel_external_make_dirs):
             fh.write('PARALLEL_DIRS += %s\n' %
                 ' '.join(obj.parallel_external_make_dirs))
 
     def _process_exports(self, obj, exports, backend_file, namespace=""):
+        # This may not be needed, but is present for backwards compatibility
+        # with the old make rules, just in case.
+        if not obj.dist_install:
+            return
+
         strings = exports.get_strings()
         if namespace:
-            if strings:
-                backend_file.write('EXPORTS_NAMESPACES += %s\n' % namespace)
-            export_name = 'EXPORTS_%s' % namespace
             namespace += '/'
-        else:
-            export_name = 'EXPORTS'
 
-        # Iterate over the list of export filenames, printing out an EXPORTS
-        # declaration for each.
-        if strings:
-            backend_file.write('%s += %s\n' % (export_name, ' '.join(strings)))
+        for s in strings:
+            source = os.path.normpath(os.path.join(obj.srcdir, s))
+            dest = '%s%s' % (namespace, os.path.basename(s))
+            self._install_manifests['dist_include'].add_symlink(source, dest)
 
-            for s in strings:
-                source = os.path.normpath(os.path.join(obj.srcdir, s))
-                if not os.path.isfile(source):
-                    raise Exception('File listed in EXPORTS does not exist: %s' % source)
-
-                p = '%s%s' % (namespace, s)
-                self._purge_manifests['dist_include'].add(p)
+            if not os.path.exists(source):
+                raise Exception('File listed in EXPORTS does not exist: %s' % source)
 
         children = exports.get_children()
         for subdir in sorted(children):
             self._process_exports(obj, children[subdir], backend_file,
                 namespace=namespace + subdir)
 
     def _handle_idl_manager(self, manager):
         build_files = self._purge_manifests['xpidl']
 
         for p in ('Makefile', 'backend.mk', '.deps/.mkdir.done',
             'headers/.mkdir.done', 'xpt/.mkdir.done'):
             build_files.add(p)
 
         for idl in manager.idls.values():
             self._install_manifests['dist_idl'].add_symlink(idl['source'],
                 idl['basename'])
-            self._purge_manifests['dist_include'].add('%s.h' % idl['root'])
+            self._install_manifests['dist_include'].add_optional_exists('%s.h'
+                % idl['root'])
             build_files.add(mozpath.join('headers', '%s.h' % idl['root']))
 
         for module in manager.modules:
             build_files.add(mozpath.join('xpt', '%s.xpt' % module))
             build_files.add(mozpath.join('.deps', '%s.pp' % module))
 
         headers = sorted('%s.h' % idl['root'] for idl in manager.idls.values())
         modules = manager.modules
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -156,21 +156,23 @@ class XPIDLFile(SandboxDerived):
 
 class Exports(SandboxDerived):
     """Sandbox container object for EXPORTS, which is a HierarchicalStringList.
 
     We need an object derived from SandboxDerived for use in the backend, so
     this object fills that role. It just has a reference to the underlying
     HierarchicalStringList, which is created when parsing EXPORTS.
     """
-    __slots__ = ('exports')
+    __slots__ = ('exports', 'dist_install')
 
-    def __init__(self, sandbox, exports):
+    def __init__(self, sandbox, exports, dist_install=True):
         SandboxDerived.__init__(self, sandbox)
         self.exports = exports
+        self.dist_install = dist_install
+
 
 class IPDLFile(SandboxDerived):
     """Describes an individual .ipdl source file."""
 
     __slots__ = (
         'basename',
     )
 
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -149,17 +149,18 @@ class TreeMetadataEmitter(LoggingMixin):
             if sandbox[moz]:
                 passthru.variables[mak] = sandbox[moz]
 
         if passthru.variables:
             yield passthru
 
         exports = sandbox.get('EXPORTS')
         if exports:
-            yield Exports(sandbox, exports)
+            yield Exports(sandbox, exports,
+                dist_install=not sandbox.get('NO_DIST_INSTALL', False))
 
         program = sandbox.get('PROGRAM')
         if program:
             yield Program(sandbox, program, sandbox['CONFIG']['BIN_SUFFIX'])
 
         for manifest in sandbox.get('XPCSHELL_TESTS_MANIFESTS', []):
             yield XpcshellManifests(sandbox, manifest)
 
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -229,43 +229,26 @@ class TestRecursiveMakeBackend(BackendTe
         }
 
         for var, val in expected.items():
             # print("test_variable_passthru[%s]" % (var))
             found = [str for str in lines if str.startswith(var)]
             self.assertEqual(found, val)
 
     def test_exports(self):
-        """Ensure EXPORTS is written out correctly."""
+        """Ensure EXPORTS is handled properly."""
         env = self._consume('exports', RecursiveMakeBackend)
 
-        backend_path = os.path.join(env.topobjdir, 'backend.mk')
-        lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
-
-        self.assertEqual(lines, [
-            'MOZBUILD_DERIVED := 1',
-            'NO_MAKEFILE_RULE := 1',
-            'NO_SUBMAKEFILES_RULE := 1',
-            'EXPORTS += foo.h',
-            'EXPORTS_NAMESPACES += mozilla',
-            'EXPORTS_mozilla += mozilla1.h mozilla2.h',
-            'EXPORTS_NAMESPACES += mozilla/dom',
-            'EXPORTS_mozilla/dom += dom1.h dom2.h',
-            'EXPORTS_NAMESPACES += mozilla/gfx',
-            'EXPORTS_mozilla/gfx += gfx.h',
-            'EXPORTS_NAMESPACES += nspr/private',
-            'EXPORTS_nspr/private += pprio.h',
-        ])
-
-        # EXPORTS files should appear in the dist_include purge manifest.
-        m = PurgeManifest(path=os.path.join(env.topobjdir,
-            '_build_manifests', 'purge', 'dist_include'))
-        self.assertIn('foo.h', m.entries)
-        self.assertIn('mozilla/mozilla1.h', m.entries)
-        self.assertIn('mozilla/dom/dom2.h', m.entries)
+        # EXPORTS files should appear in the dist_include install manifest.
+        m = InstallManifest(path=os.path.join(env.topobjdir,
+            '_build_manifests', 'install', 'dist_include'))
+        self.assertEqual(len(m), 7)
+        self.assertIn('foo.h', m)
+        self.assertIn('mozilla/mozilla1.h', m)
+        self.assertIn('mozilla/dom/dom2.h', m)
 
     def test_xpcshell_manifests(self):
         """Ensure XPCSHELL_TESTS_MANIFESTS is written out correctly."""
         env = self._consume('xpcshell_manifests', RecursiveMakeBackend)
 
         backend_path = os.path.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
@@ -292,18 +275,18 @@ class TestRecursiveMakeBackend(BackendTe
         self.assertIn('.deps/my_module.pp', m.entries)
         self.assertIn('xpt/my_module.xpt', m.entries)
 
         m = InstallManifest(path=os.path.join(install_dir, 'dist_idl'))
         self.assertEqual(len(m), 2)
         self.assertIn('bar.idl', m)
         self.assertIn('foo.idl', m)
 
-        m = PurgeManifest(path=os.path.join(purge_dir, 'dist_include'))
-        self.assertIn('foo.h', m.entries)
+        m = InstallManifest(path=os.path.join(install_dir, 'dist_include'))
+        self.assertIn('foo.h', m)
 
         p = os.path.join(env.topobjdir, 'config/makefiles/xpidl')
         self.assertTrue(os.path.isdir(p))
 
         self.assertTrue(os.path.isfile(os.path.join(p, 'Makefile')))
 
     def test_xpcshell_master_manifest(self):
         """Ensure that the master xpcshell manifest is written out correctly."""
@@ -320,17 +303,16 @@ class TestRecursiveMakeBackend(BackendTe
     def test_purge_manifests_written(self):
         env = self._consume('stub0', RecursiveMakeBackend)
 
         purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
         self.assertTrue(os.path.exists(purge_dir))
 
         expected = [
             'dist_bin',
-            'dist_include',
             'dist_private',
             'dist_public',
             'dist_sdk',
             'tests',
         ]
 
         for e in expected:
             full = os.path.join(purge_dir, e)