Bug 911375 - Part 2: Convert uses of PurgeManifest to InstallManifest; r=glandium
authorGregory Szorc <gps@mozilla.com>
Mon, 16 Sep 2013 17:49:44 -0700
changeset 162784 43b40211d0dac8cec7fbb869b7da154fe2582c88
parent 162783 9f2d0e38fdf85fd768afc7d998e796f88fe6882c
child 162785 b748f8af804c5c8f794f3c8b544b35079b416d5b
push id428
push userbbajaj@mozilla.com
push dateTue, 28 Jan 2014 00:16:25 +0000
treeherdermozilla-release@cd72a7ff3a75 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs911375
milestone27.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 911375 - Part 2: Convert uses of PurgeManifest to InstallManifest; r=glandium
Makefile.in
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -26,42 +26,19 @@ include $(topsrcdir)/config/config.mk
 GARBAGE_DIRS += dist _javagen _profile _tests staticlib
 DIST_GARBAGE = config.cache config.log config.status* config-defs.h \
    config/autoconf.mk \
    mozilla-config.h \
    netwerk/necko-config.h xpcom/xpcom-config.h xpcom/xpcom-private.h \
    .mozconfig.mk
 
 ifndef MOZ_PROFILE_USE
-# One of the first things we do in the build is purge "unknown" files
-# from the object directory. This serves two purposes:
-#
-#   1) Remove files from a previous build no longer accounted for in
-#      this build configuration.
-#
-#   2) Work around poor build system dependencies by forcing some
-#      rebuilds.
-#
-# Ideally #2 does not exist. Our reliance on this aspect should diminish
-# over time.
-#
-# moz.build backend generation simply installs a set of "manifests" into
-# a common directory. Each manifest is responsible for defining files in
-# a specific subdirectory of the object directory. The invoked Python
-# script simply iterates over all the manifests, purging files as
-# necessary. To manage new directories or add files to the manifests,
-# modify the backend generator.
-#
 # We need to explicitly put backend.RecursiveMakeBackend.built here
 # otherwise the rule in rules.mk doesn't run early enough.
-libs export tools:: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built
-	$(call SUBMAKE,backend.RecursiveMakeBackend.built,js/src,1)
-
-export::
-	$(call py_action,purge_manifests,-d _build_manifests/purge .)
+libs export tools:: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built js-config-status
 endif
 
 CLOBBER: $(topsrcdir)/CLOBBER
 	@echo "STOP!  The CLOBBER file has changed."
 	@echo "Please run the build through a sanctioned build wrapper, such as"
 	@echo "'mach build' or client.mk."
 	@exit 1
 
@@ -74,27 +51,63 @@ CLOBBER: $(topsrcdir)/CLOBBER
 
 config.status: $(topsrcdir)/configure
 	@echo "STOP!  configure has changed and needs to be run in this build directory."
 	@echo "Please rerun configure."
 	@echo "To ignore this message, touch 'config.status' in the build directory,"
 	@echo "but your build might not succeed."
 	@exit 1
 
-export::
-	$(RM) -r $(DIST)/sdk
+.PHONY: js-config-status
+js-config-status:
+	$(call SUBMAKE,backend.RecursiveMakeBackend.built,js/src,1)
+
+install_manifests := bin idl include public private sdk
+install_manifest_depends = \
+  CLOBBER \
+  $(topsrcdir)/configure \
+  config.status \
+  backend.RecursiveMakeBackend.built \
+  js-config-status \
+  $(NULL)
+
+.PHONY: install-manifests
+install-manifests: $(addprefix install-dist-,$(install_manifests))
+
+.PHONY: $(addprefix install-dist-,$(install_manifests))
+$(addprefix install-dist-,$(install_manifests)): install-dist-%: $(install_manifest_depends)
+	$(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )$(DIST)/$* _build_manifests/install/dist_$* js/src/_build_manifests/install/dist_$*)
+
+.PHONY: install-tests
+install-manifests: install-tests
+install-tests: $(install_manifest_depends)
+	$(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )_tests _build_manifests/install/tests js/src/_build_manifests/install/tests)
+
+# Windows PGO builds don't perform a clean before the 2nd pass. So, we want
+# to preserve content for the 2nd pass on Windows. Everywhere else, we always
+# process the install manifests as part of export.
+ifdef MOZ_PROFILE_USE
+ifndef NO_PROFILE_GUIDED_OPTIMIZE
+ifneq ($(OS_ARCH)_$(GNU_CC), WINNT_)
+export:: install-manifests
+endif
+endif
+else # !MOZ_PROFILE_USE (normal build)
+export:: install-manifests
+endif
+
+# For historical reasons that are unknown, $(DIST)/sdk is always blown away
+# with no regard for PGO passes. This decision could probably be revisited.
+export:: install-dist-sdk
 
 ifdef ENABLE_TESTS
 # Additional makefile targets to call automated test suites
 include $(topsrcdir)/testing/testsuite-targets.mk
 endif
 
-export::
-	$(call py_action,process_install_manifest,$(DIST)/include _build_manifests/install/dist_include js/src/_build_manifests/install/dist_include)
-
 default all::
 	$(call BUILDSTATUS,TIERS export compile libs tools)
 
 include $(topsrcdir)/config/rules.mk
 
 distclean::
 	$(RM) $(DIST_GARBAGE)
 
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -7,17 +7,16 @@ from __future__ import unicode_literals
 import errno
 import logging
 import os
 import types
 
 from mozpack.copier import FilePurger
 from mozpack.manifests import (
     InstallManifest,
-    PurgeManifest,
 )
 import mozpack.path as mozpath
 
 from .common import CommonBackend
 from ..frontend.data import (
     ConfigFileSubstitution,
     Defines,
     DirectoryTraversal,
@@ -143,29 +142,27 @@ class RecursiveMakeBackend(CommonBackend
         self.summary.backend_detailed_summary = types.MethodType(detailed,
             self.summary)
 
         self.xpcshell_manifests = []
 
         self.backend_input_files.add(os.path.join(self.environment.topobjdir,
             'config', 'autoconf.mk'))
 
-        self._purge_manifests = dict(
-            dist_bin=PurgeManifest(relpath='dist/bin'),
-            dist_private=PurgeManifest(relpath='dist/private'),
-            dist_public=PurgeManifest(relpath='dist/public'),
-            dist_sdk=PurgeManifest(relpath='dist/sdk'),
-            tests=PurgeManifest(relpath='_tests'),
-            xpidl=PurgeManifest(relpath='config/makefiles/xpidl'),
-        )
-
-        self._install_manifests = dict(
-            dist_idl=InstallManifest(),
-            dist_include=InstallManifest(),
-        )
+        self._install_manifests = {
+            k: InstallManifest() for k in [
+                'dist_bin',
+                'dist_idl',
+                'dist_include',
+                'dist_public',
+                'dist_private',
+                'dist_sdk',
+                'tests',
+                'xpidl',
+            ]}
 
     def _update_from_avoid_write(self, result):
         existed, updated = result
 
         if not existed:
             self.summary.created_count += 1
         elif updated:
             self.summary.updated_count += 1
@@ -358,17 +355,16 @@ class RecursiveMakeBackend(CommonBackend
             mastermanifest.write(
                 '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.\n\n')
             for manifest in self.xpcshell_manifests:
                 mastermanifest.write("[include:%s]\n" % manifest)
             self._update_from_avoid_write(mastermanifest.close())
             self.summary.managed_count += 1
 
         self._write_manifests('install', self._install_manifests)
-        self._write_manifests('purge', self._purge_manifests)
 
     def _process_directory_traversal(self, obj, backend_file):
         """Process a data.DirectoryTraversal instance."""
         fh = backend_file.fh
 
         for tier, dirs in obj.tier_dirs.iteritems():
             fh.write('TIERS += %s\n' % tier)
 
@@ -427,31 +423,33 @@ class RecursiveMakeBackend(CommonBackend
                 raise Exception('File listed in EXPORTS does not exist: %s' % source)
 
         children = exports.get_children()
         for subdir in sorted(children):
             self._process_exports(obj, children[subdir], backend_file,
                 namespace=namespace + subdir)
 
     def _handle_idl_manager(self, manager):
-        build_files = self._purge_manifests['xpidl']
+        build_files = self._install_manifests['xpidl']
 
         for p in ('Makefile', 'backend.mk', '.deps/.mkdir.done',
             'xpt/.mkdir.done'):
-            build_files.add(p)
+            build_files.add_optional_exists(p)
 
         for idl in manager.idls.values():
             self._install_manifests['dist_idl'].add_symlink(idl['source'],
                 idl['basename'])
             self._install_manifests['dist_include'].add_optional_exists('%s.h'
                 % idl['root'])
 
         for module in manager.modules:
-            build_files.add(mozpath.join('xpt', '%s.xpt' % module))
-            build_files.add(mozpath.join('.deps', '%s.pp' % module))
+            build_files.add_optional_exists(mozpath.join('xpt',
+                '%s.xpt' % module))
+            build_files.add_optional_exists(mozpath.join('.deps',
+                '%s.pp' % module))
 
         modules = manager.modules
         xpt_modules = sorted(modules.keys())
         rules = []
 
         for module in xpt_modules:
             deps = sorted(modules[module])
             idl_deps = ['$(dist_idl_dir)/%s.idl' % dep for dep in deps]
@@ -470,17 +468,17 @@ class RecursiveMakeBackend(CommonBackend
                 '\t@echo "$(notdir $@)"',
                 '\t$(idlprocess) $(basename $(notdir $@)) %s' % ' '.join(deps),
                 '',
             ])
 
         # Create dependency for output header so we force regeneration if the
         # header was deleted. This ideally should not be necessary. However,
         # some processes (such as PGO at the time this was implemented) wipe
-        # out dist/include without regard to our install/purge manifests.
+        # out dist/include without regard to our install manifests.
 
         out_path = os.path.join(self.environment.topobjdir, 'config',
             'makefiles', 'xpidl', 'Makefile')
         result = self.environment.create_config_file(out_path, extra=dict(
             xpidl_rules='\n'.join(rules),
             xpidl_modules=' '.join(xpt_modules),
         ))
         self._update_from_avoid_write(result)
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -3,17 +3,16 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 import os
 
 from mozpack.manifests import (
     InstallManifest,
-    PurgeManifest,
 )
 from mozunit import main
 
 from mozbuild.backend.recursivemake import RecursiveMakeBackend
 from mozbuild.frontend.emitter import TreeMetadataEmitter
 from mozbuild.frontend.reader import BuildReader
 
 from mozbuild.test.backend.common import BackendTester
@@ -262,32 +261,31 @@ class TestRecursiveMakeBackend(BackendTe
         # Assignment[aa], append[cc], conditional[valid]
         expected = ('aa', 'bb', 'cc', 'dd', 'valid_val')
         self.assertEqual(xpclines, ["XPCSHELL_TESTS += %s" % val for val in expected])
 
     def test_xpidl_generation(self):
         """Ensure xpidl files and directories are written out."""
         env = self._consume('xpidl', RecursiveMakeBackend)
 
-        # Purge manifests should contain entries.
-        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
+        # Install manifests should contain entries.
         install_dir = os.path.join(env.topobjdir, '_build_manifests',
             'install')
-        self.assertTrue(os.path.isfile(os.path.join(purge_dir, 'xpidl')))
         self.assertTrue(os.path.isfile(os.path.join(install_dir, 'dist_idl')))
-
-        m = PurgeManifest(path=os.path.join(purge_dir, 'xpidl'))
-        self.assertIn('.deps/my_module.pp', m.entries)
-        self.assertIn('xpt/my_module.xpt', m.entries)
+        self.assertTrue(os.path.isfile(os.path.join(install_dir, 'xpidl')))
 
         m = InstallManifest(path=os.path.join(install_dir, 'dist_idl'))
         self.assertEqual(len(m), 2)
         self.assertIn('bar.idl', m)
         self.assertIn('foo.idl', m)
 
+        m = InstallManifest(path=os.path.join(install_dir, 'xpidl'))
+        self.assertIn('.deps/my_module.pp', m)
+        self.assertIn('xpt/my_module.xpt', m)
+
         m = InstallManifest(path=os.path.join(install_dir, 'dist_include'))
         self.assertIn('foo.h', m)
 
         p = os.path.join(env.topobjdir, 'config/makefiles/xpidl')
         self.assertTrue(os.path.isdir(p))
 
         self.assertTrue(os.path.isfile(os.path.join(p, 'Makefile')))
 
@@ -298,45 +296,24 @@ class TestRecursiveMakeBackend(BackendTe
         manifest_path = os.path.join(env.topobjdir,
             'testing', 'xpcshell', 'xpcshell.ini')
         lines = [l.strip() for l in open(manifest_path, 'rt').readlines()]
         expected = ('aa', 'bb', 'cc', 'dd', 'valid_val')
         self.assertEqual(lines, [
             '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.',
             ''] + ['[include:%s/xpcshell.ini]' % x for x in expected])
 
-    def test_purge_manifests_written(self):
-        env = self._consume('stub0', RecursiveMakeBackend)
-
-        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
-        self.assertTrue(os.path.exists(purge_dir))
-
-        expected = [
-            'dist_bin',
-            'dist_private',
-            'dist_public',
-            'dist_sdk',
-            'tests',
-        ]
-
-        for e in expected:
-            full = os.path.join(purge_dir, e)
-            self.assertTrue(os.path.exists(full))
-
-        m = PurgeManifest(path=os.path.join(purge_dir, 'dist_bin'))
-        self.assertEqual(m.relpath, 'dist/bin')
-
-    def test_old_purge_manifest_deleted(self):
-        # Simulate a purge manifest from a previous backend version. Ensure it
-        # is deleted.
+    def test_old_install_manifest_deleted(self):
+        # Simulate an install manifest from a previous backend version. Ensure
+        # it is deleted.
         env = self._get_environment('stub0')
-        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
+        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'install')
         manifest_path = os.path.join(purge_dir, 'old_manifest')
         os.makedirs(purge_dir)
-        m = PurgeManifest()
+        m = InstallManifest()
         m.write(path=manifest_path)
 
         self.assertTrue(os.path.exists(manifest_path))
         self._consume('stub0', RecursiveMakeBackend, env)
         self.assertFalse(os.path.exists(manifest_path))
 
     def test_install_manifests_written(self):
         env, objs = self._emit('stub0')