Bug 911375 - Part 2: Convert uses of PurgeManifest to InstallManifest; r=glandium
authorGregory Szorc <gps@mozilla.com>
Mon, 16 Sep 2013 17:49:44 -0700
changeset 161832 43b40211d0dac8cec7fbb869b7da154fe2582c88
parent 161831 9f2d0e38fdf85fd768afc7d998e796f88fe6882c
child 161833 b748f8af804c5c8f794f3c8b544b35079b416d5b
push id1
push usersledru@mozilla.com
push dateThu, 04 Dec 2014 17:57:20 +0000
reviewersglandium
bugs911375
milestone27.0a1
Bug 911375 - Part 2: Convert uses of PurgeManifest to InstallManifest; r=glandium
Makefile.in
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -26,42 +26,19 @@ include $(topsrcdir)/config/config.mk
 GARBAGE_DIRS += dist _javagen _profile _tests staticlib
 DIST_GARBAGE = config.cache config.log config.status* config-defs.h \
    config/autoconf.mk \
    mozilla-config.h \
    netwerk/necko-config.h xpcom/xpcom-config.h xpcom/xpcom-private.h \
    .mozconfig.mk
 
 ifndef MOZ_PROFILE_USE
-# One of the first things we do in the build is purge "unknown" files
-# from the object directory. This serves two purposes:
-#
-#   1) Remove files from a previous build no longer accounted for in
-#      this build configuration.
-#
-#   2) Work around poor build system dependencies by forcing some
-#      rebuilds.
-#
-# Ideally #2 does not exist. Our reliance on this aspect should diminish
-# over time.
-#
-# moz.build backend generation simply installs a set of "manifests" into
-# a common directory. Each manifest is responsible for defining files in
-# a specific subdirectory of the object directory. The invoked Python
-# script simply iterates over all the manifests, purging files as
-# necessary. To manage new directories or add files to the manifests,
-# modify the backend generator.
-#
 # We need to explicitly put backend.RecursiveMakeBackend.built here
 # otherwise the rule in rules.mk doesn't run early enough.
-libs export tools:: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built
-	$(call SUBMAKE,backend.RecursiveMakeBackend.built,js/src,1)
-
-export::
-	$(call py_action,purge_manifests,-d _build_manifests/purge .)
+libs export tools:: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built js-config-status
 endif
 
 CLOBBER: $(topsrcdir)/CLOBBER
 	@echo "STOP!  The CLOBBER file has changed."
 	@echo "Please run the build through a sanctioned build wrapper, such as"
 	@echo "'mach build' or client.mk."
 	@exit 1
 
@@ -74,27 +51,63 @@ CLOBBER: $(topsrcdir)/CLOBBER
 
 config.status: $(topsrcdir)/configure
 	@echo "STOP!  configure has changed and needs to be run in this build directory."
 	@echo "Please rerun configure."
 	@echo "To ignore this message, touch 'config.status' in the build directory,"
 	@echo "but your build might not succeed."
 	@exit 1
 
-export::
-	$(RM) -r $(DIST)/sdk
+.PHONY: js-config-status
+js-config-status:
+	$(call SUBMAKE,backend.RecursiveMakeBackend.built,js/src,1)
+
+install_manifests := bin idl include public private sdk
+install_manifest_depends = \
+  CLOBBER \
+  $(topsrcdir)/configure \
+  config.status \
+  backend.RecursiveMakeBackend.built \
+  js-config-status \
+  $(NULL)
+
+.PHONY: install-manifests
+install-manifests: $(addprefix install-dist-,$(install_manifests))
+
+.PHONY: $(addprefix install-dist-,$(install_manifests))
+$(addprefix install-dist-,$(install_manifests)): install-dist-%: $(install_manifest_depends)
+	$(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )$(DIST)/$* _build_manifests/install/dist_$* js/src/_build_manifests/install/dist_$*)
+
+.PHONY: install-tests
+install-manifests: install-tests
+install-tests: $(install_manifest_depends)
+	$(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )_tests _build_manifests/install/tests js/src/_build_manifests/install/tests)
+
+# Windows PGO builds don't perform a clean before the 2nd pass. So, we want
+# to preserve content for the 2nd pass on Windows. Everywhere else, we always
+# process the install manifests as part of export.
+ifdef MOZ_PROFILE_USE
+ifndef NO_PROFILE_GUIDED_OPTIMIZE
+ifneq ($(OS_ARCH)_$(GNU_CC), WINNT_)
+export:: install-manifests
+endif
+endif
+else # !MOZ_PROFILE_USE (normal build)
+export:: install-manifests
+endif
+
+# For historical reasons that are unknown, $(DIST)/sdk is always blown away
+# with no regard for PGO passes. This decision could probably be revisited.
+export:: install-dist-sdk
 
 ifdef ENABLE_TESTS
 # Additional makefile targets to call automated test suites
 include $(topsrcdir)/testing/testsuite-targets.mk
 endif
 
-export::
-	$(call py_action,process_install_manifest,$(DIST)/include _build_manifests/install/dist_include js/src/_build_manifests/install/dist_include)
-
 default all::
 	$(call BUILDSTATUS,TIERS export compile libs tools)
 
 include $(topsrcdir)/config/rules.mk
 
 distclean::
 	$(RM) $(DIST_GARBAGE)
 
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -7,17 +7,16 @@ from __future__ import unicode_literals
 import errno
 import logging
 import os
 import types
 
 from mozpack.copier import FilePurger
 from mozpack.manifests import (
     InstallManifest,
-    PurgeManifest,
 )
 import mozpack.path as mozpath
 
 from .common import CommonBackend
 from ..frontend.data import (
     ConfigFileSubstitution,
     Defines,
     DirectoryTraversal,
@@ -143,29 +142,27 @@ class RecursiveMakeBackend(CommonBackend
         self.summary.backend_detailed_summary = types.MethodType(detailed,
             self.summary)
 
         self.xpcshell_manifests = []
 
         self.backend_input_files.add(os.path.join(self.environment.topobjdir,
             'config', 'autoconf.mk'))
 
-        self._purge_manifests = dict(
-            dist_bin=PurgeManifest(relpath='dist/bin'),
-            dist_private=PurgeManifest(relpath='dist/private'),
-            dist_public=PurgeManifest(relpath='dist/public'),
-            dist_sdk=PurgeManifest(relpath='dist/sdk'),
-            tests=PurgeManifest(relpath='_tests'),
-            xpidl=PurgeManifest(relpath='config/makefiles/xpidl'),
-        )
-
-        self._install_manifests = dict(
-            dist_idl=InstallManifest(),
-            dist_include=InstallManifest(),
-        )
+        self._install_manifests = {
+            k: InstallManifest() for k in [
+                'dist_bin',
+                'dist_idl',
+                'dist_include',
+                'dist_public',
+                'dist_private',
+                'dist_sdk',
+                'tests',
+                'xpidl',
+            ]}
 
     def _update_from_avoid_write(self, result):
         existed, updated = result
 
         if not existed:
             self.summary.created_count += 1
         elif updated:
             self.summary.updated_count += 1
@@ -358,17 +355,16 @@ class RecursiveMakeBackend(CommonBackend
             mastermanifest.write(
                 '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.\n\n')
             for manifest in self.xpcshell_manifests:
                 mastermanifest.write("[include:%s]\n" % manifest)
             self._update_from_avoid_write(mastermanifest.close())
             self.summary.managed_count += 1
 
         self._write_manifests('install', self._install_manifests)
-        self._write_manifests('purge', self._purge_manifests)
 
     def _process_directory_traversal(self, obj, backend_file):
         """Process a data.DirectoryTraversal instance."""
         fh = backend_file.fh
 
         for tier, dirs in obj.tier_dirs.iteritems():
             fh.write('TIERS += %s\n' % tier)
 
@@ -427,31 +423,33 @@ class RecursiveMakeBackend(CommonBackend
                 raise Exception('File listed in EXPORTS does not exist: %s' % source)
 
         children = exports.get_children()
         for subdir in sorted(children):
             self._process_exports(obj, children[subdir], backend_file,
                 namespace=namespace + subdir)
 
     def _handle_idl_manager(self, manager):
-        build_files = self._purge_manifests['xpidl']
+        build_files = self._install_manifests['xpidl']
 
         for p in ('Makefile', 'backend.mk', '.deps/.mkdir.done',
             'xpt/.mkdir.done'):
-            build_files.add(p)
+            build_files.add_optional_exists(p)
 
         for idl in manager.idls.values():
             self._install_manifests['dist_idl'].add_symlink(idl['source'],
                 idl['basename'])
             self._install_manifests['dist_include'].add_optional_exists('%s.h'
                 % idl['root'])
 
         for module in manager.modules:
-            build_files.add(mozpath.join('xpt', '%s.xpt' % module))
-            build_files.add(mozpath.join('.deps', '%s.pp' % module))
+            build_files.add_optional_exists(mozpath.join('xpt',
+                '%s.xpt' % module))
+            build_files.add_optional_exists(mozpath.join('.deps',
+                '%s.pp' % module))
 
         modules = manager.modules
         xpt_modules = sorted(modules.keys())
         rules = []
 
         for module in xpt_modules:
             deps = sorted(modules[module])
             idl_deps = ['$(dist_idl_dir)/%s.idl' % dep for dep in deps]
@@ -470,17 +468,17 @@ class RecursiveMakeBackend(CommonBackend
                 '\t@echo "$(notdir $@)"',
                 '\t$(idlprocess) $(basename $(notdir $@)) %s' % ' '.join(deps),
                 '',
             ])
 
         # Create dependency for output header so we force regeneration if the
         # header was deleted. This ideally should not be necessary. However,
         # some processes (such as PGO at the time this was implemented) wipe
-        # out dist/include without regard to our install/purge manifests.
+        # out dist/include without regard to our install manifests.
 
         out_path = os.path.join(self.environment.topobjdir, 'config',
             'makefiles', 'xpidl', 'Makefile')
         result = self.environment.create_config_file(out_path, extra=dict(
             xpidl_rules='\n'.join(rules),
             xpidl_modules=' '.join(xpt_modules),
         ))
         self._update_from_avoid_write(result)
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -3,17 +3,16 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 import os
 
 from mozpack.manifests import (
     InstallManifest,
-    PurgeManifest,
 )
 from mozunit import main
 
 from mozbuild.backend.recursivemake import RecursiveMakeBackend
 from mozbuild.frontend.emitter import TreeMetadataEmitter
 from mozbuild.frontend.reader import BuildReader
 
 from mozbuild.test.backend.common import BackendTester
@@ -262,32 +261,31 @@ class TestRecursiveMakeBackend(BackendTe
         # Assignment[aa], append[cc], conditional[valid]
         expected = ('aa', 'bb', 'cc', 'dd', 'valid_val')
         self.assertEqual(xpclines, ["XPCSHELL_TESTS += %s" % val for val in expected])
 
     def test_xpidl_generation(self):
         """Ensure xpidl files and directories are written out."""
         env = self._consume('xpidl', RecursiveMakeBackend)
 
-        # Purge manifests should contain entries.
-        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
+        # Install manifests should contain entries.
         install_dir = os.path.join(env.topobjdir, '_build_manifests',
             'install')
-        self.assertTrue(os.path.isfile(os.path.join(purge_dir, 'xpidl')))
         self.assertTrue(os.path.isfile(os.path.join(install_dir, 'dist_idl')))
-
-        m = PurgeManifest(path=os.path.join(purge_dir, 'xpidl'))
-        self.assertIn('.deps/my_module.pp', m.entries)
-        self.assertIn('xpt/my_module.xpt', m.entries)
+        self.assertTrue(os.path.isfile(os.path.join(install_dir, 'xpidl')))
 
         m = InstallManifest(path=os.path.join(install_dir, 'dist_idl'))
         self.assertEqual(len(m), 2)
         self.assertIn('bar.idl', m)
         self.assertIn('foo.idl', m)
 
+        m = InstallManifest(path=os.path.join(install_dir, 'xpidl'))
+        self.assertIn('.deps/my_module.pp', m)
+        self.assertIn('xpt/my_module.xpt', m)
+
         m = InstallManifest(path=os.path.join(install_dir, 'dist_include'))
         self.assertIn('foo.h', m)
 
         p = os.path.join(env.topobjdir, 'config/makefiles/xpidl')
         self.assertTrue(os.path.isdir(p))
 
         self.assertTrue(os.path.isfile(os.path.join(p, 'Makefile')))
 
@@ -298,45 +296,24 @@ class TestRecursiveMakeBackend(BackendTe
         manifest_path = os.path.join(env.topobjdir,
             'testing', 'xpcshell', 'xpcshell.ini')
         lines = [l.strip() for l in open(manifest_path, 'rt').readlines()]
         expected = ('aa', 'bb', 'cc', 'dd', 'valid_val')
         self.assertEqual(lines, [
             '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.',
             ''] + ['[include:%s/xpcshell.ini]' % x for x in expected])
 
-    def test_purge_manifests_written(self):
-        env = self._consume('stub0', RecursiveMakeBackend)
-
-        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
-        self.assertTrue(os.path.exists(purge_dir))
-
-        expected = [
-            'dist_bin',
-            'dist_private',
-            'dist_public',
-            'dist_sdk',
-            'tests',
-        ]
-
-        for e in expected:
-            full = os.path.join(purge_dir, e)
-            self.assertTrue(os.path.exists(full))
-
-        m = PurgeManifest(path=os.path.join(purge_dir, 'dist_bin'))
-        self.assertEqual(m.relpath, 'dist/bin')
-
-    def test_old_purge_manifest_deleted(self):
-        # Simulate a purge manifest from a previous backend version. Ensure it
-        # is deleted.
+    def test_old_install_manifest_deleted(self):
+        # Simulate an install manifest from a previous backend version. Ensure
+        # it is deleted.
         env = self._get_environment('stub0')
-        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
+        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'install')
         manifest_path = os.path.join(purge_dir, 'old_manifest')
         os.makedirs(purge_dir)
-        m = PurgeManifest()
+        m = InstallManifest()
         m.write(path=manifest_path)
 
         self.assertTrue(os.path.exists(manifest_path))
         self._consume('stub0', RecursiveMakeBackend, env)
         self.assertFalse(os.path.exists(manifest_path))
 
     def test_install_manifests_written(self):
         env, objs = self._emit('stub0')