Bug 1390916 - Always use --track instead of --no-remove for install manifests; r=gps
authorMike Shal <mshal@mozilla.com>
Tue, 08 Aug 2017 17:07:07 -0400
changeset 376069 3a181cd6052b21fb0aa1a5a855aa5cff81466862
parent 376068 1b502238070fdbd7b887f2749013c9b59929779c
child 376070 c7e972957ff0d3f8d427c827d0536757055420df
push id94023
push userarchaeopteryx@coole-files.de
push dateTue, 22 Aug 2017 09:51:45 +0000
treeherdermozilla-inbound@2b5a763e1ccd [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1390916
milestone57.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1390916 - Always use --track instead of --no-remove for install manifests; r=gps The --track flag provides a more accurate accounting of what files were installed by the manifest, so they can be appropriately removed. For example, test files are now removed from _tests if an entry in a test file is deleted. The --no-remove flag is removed as an alternative, and the --track flag is now mandatory. MozReview-Commit-ID: Wiup4Gzwkb
Makefile.in
js/src/build/Makefile.in
python/mozbuild/mozbuild/action/process_install_manifest.py
xpcom/xpidl/Makefile.in
--- a/Makefile.in
+++ b/Makefile.in
@@ -155,51 +155,35 @@ ifdef MOZ_ARTIFACT_BUILDS
 endif
 	$(call BUILDSTATUS,TIER_START make)
 	$(MAKE) buildid.h source-repo.h
 	$(call BUILDSTATUS,TIER_FINISH make)
 	$(call BUILDSTATUS,TIER_START tup)
 	@$(TUP) $(if $(findstring s,$(filter-out --%,$(MAKEFLAGS))),,--verbose)
 	$(call BUILDSTATUS,TIER_FINISH tup)
 
-# process_install_manifest needs to be invoked with --no-remove when building
-# js as standalone because automated builds are building nspr separately and
-# that would remove the resulting files.
-# Eventually, a standalone js build would just be able to build nspr itself,
-# removing the need for the former.
-ifdef JS_STANDALONE
-NO_REMOVE=1
-endif
-
-# For an artifact build, _tests will already be partly populated, so run
-# this install manifest with NO_REMOVE set in this case.
-ifdef MOZ_ARTIFACT_BUILDS
-install-_tests: NO_REMOVE=1
-endif
-
 .PHONY: $(addprefix install-,$(install_manifests))
 $(addprefix install-,$(install_manifests)): install-%: $(install_manifest_depends)
 ifneq (,$(filter FasterMake+RecursiveMake,$(BUILD_BACKENDS)))
 	@# If we're using the hybrid FasterMake/RecursiveMake backend, we want
 	@# to ensure the FasterMake end doesn't have install manifests for the
 	@# same directory, because that would blow up
 	$(if $(wildcard _build_manifests/install/$(subst /,_,$*)),$(if $(wildcard faster/install_$(subst /,_,$*)*),$(error FasterMake and RecursiveMake ends of the hybrid build system want to handle $*)))
 endif
-	$(addprefix $(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )$*) ,$(wildcard _build_manifests/install/$(subst /,_,$*)))
+	$(addprefix $(call py_action,process_install_manifest,--track install_$(subst /,_,$*).track $*) ,$(wildcard _build_manifests/install/$(subst /,_,$*)))
 
 # Dummy wrapper rule to allow the faster backend to piggy back
 $(addprefix install-,$(subst /,_,$(filter dist/%,$(install_manifests)))): install-dist_%: install-dist/% ;
 
 .PHONY: install-tests
 install-tests: install-test-files
 
-# Force --no-remove, because $objdir/_tests is handled by multiple manifests.
 .PHONY: install-test-files
 install-test-files:
-	$(call py_action,process_install_manifest,--no-remove _tests _build_manifests/install/_test_files)
+	$(call py_action,process_install_manifest,--track install__test_files.track _tests _build_manifests/install/_test_files)
 
 include $(topsrcdir)/build/moz-automation.mk
 
 # dist and _tests should be purged during cleaning. However, we don't want them
 # purged during PGO builds because they contain some auto-generated files.
 ifneq ($(filter-out maybe_clobber_profiledbuild,$(MAKECMDGOALS)),)
 GARBAGE_DIRS += dist _tests
 endif
@@ -212,23 +196,23 @@ recurse_pre-export::
 # process the install manifests as part of export.
 # For the binaries rule, not all the install manifests matter, so force only
 # the interesting ones to be done.
 ifdef MOZ_PROFILE_USE
 ifndef NO_PROFILE_GUIDED_OPTIMIZE
 ifneq ($(OS_ARCH)_$(GNU_CC), WINNT_)
 recurse_pre-export:: install-manifests
 binaries::
-	@$(MAKE) install-manifests NO_REMOVE=1 install_manifests=dist/include
+	@$(MAKE) install-manifests install_manifests=dist/include
 endif
 endif
 else # !MOZ_PROFILE_USE (normal build)
 recurse_pre-export:: install-manifests
 binaries::
-	@$(MAKE) install-manifests NO_REMOVE=1 install_manifests=dist/include
+	@$(MAKE) install-manifests install_manifests=dist/include
 endif
 
 recurse_artifact:
 	$(topsrcdir)/mach --log-no-times artifact install
 
 ifndef JS_STANDALONE
 ifdef ENABLE_TESTS
 # Additional makefile targets to call automated test suites
--- a/js/src/build/Makefile.in
+++ b/js/src/build/Makefile.in
@@ -65,17 +65,17 @@ install:: ../js-config.h
 #          js/src/ds/* public headers
 #        gc/
 #          js/src/gc/* public headers
 #        mozilla/
 #          mfbt headers
 #
 
 install::
-	$(call py_action,process_install_manifest,--no-remove --no-symlinks $(DESTDIR)$(includedir) $(DEPTH)/_build_manifests/install/dist_include)
+	$(call py_action,process_install_manifest,--track install_dist_include.track --no-symlinks $(DESTDIR)$(includedir) $(DEPTH)/_build_manifests/install/dist_include)
 
 #
 # END SpiderMonkey header installation
 #############################################
 
 # Install versioned script, for parallel installability in Linux distributions
 install:: js-config
 	cp $^ js$(MOZJS_MAJOR_VERSION)-config
--- a/python/mozbuild/mozbuild/action/process_install_manifest.py
+++ b/python/mozbuild/mozbuild/action/process_install_manifest.py
@@ -23,42 +23,41 @@ from mozpack.manifests import (
 from mozbuild.util import DefinesAction
 
 
 COMPLETE = 'Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; ' \
     'Added/updated {updated}; ' \
     'Removed {rm_files} files and {rm_dirs} directories.'
 
 
-def process_manifest(destdir, paths, track=None,
-        remove_unaccounted=True,
-        remove_all_directory_symlinks=True,
-        remove_empty_directories=True,
+def process_manifest(destdir, paths, track,
         no_symlinks=False,
         defines={}):
 
-    if track:
-        if os.path.exists(track):
-            # We use the same format as install manifests for the tracking
-            # data.
-            manifest = InstallManifest(path=track)
-            remove_unaccounted = FileRegistry()
-            dummy_file = BaseFile()
+    if os.path.exists(track):
+        # We use the same format as install manifests for the tracking
+        # data.
+        manifest = InstallManifest(path=track)
+        remove_unaccounted = FileRegistry()
+        dummy_file = BaseFile()
 
-            finder = FileFinder(destdir, find_dotfiles=True)
-            for dest in manifest._dests:
-                for p, f in finder.find(dest):
-                    remove_unaccounted.add(p, dummy_file)
+        finder = FileFinder(destdir, find_dotfiles=True)
+        for dest in manifest._dests:
+            for p, f in finder.find(dest):
+                remove_unaccounted.add(p, dummy_file)
 
-        else:
-            # If tracking is enabled and there is no file, we don't want to
-            # be removing anything.
-            remove_unaccounted=False
-            remove_empty_directories=False
-            remove_all_directory_symlinks=False
+        remove_empty_directories=True
+        remove_all_directory_symlinks=True
+
+    else:
+        # If tracking is enabled and there is no file, we don't want to
+        # be removing anything.
+        remove_unaccounted = False
+        remove_empty_directories=False
+        remove_all_directory_symlinks=False
 
     manifest = InstallManifest()
     for path in paths:
         manifest |= InstallManifest(path=path)
 
     copier = FileCopier()
     link_policy = "copy" if no_symlinks else "symlink"
     manifest.populate_registry(
@@ -76,38 +75,30 @@ def process_manifest(destdir, paths, tra
 
 
 def main(argv):
     parser = argparse.ArgumentParser(
         description='Process install manifest files.')
 
     parser.add_argument('destdir', help='Destination directory.')
     parser.add_argument('manifests', nargs='+', help='Path to manifest file(s).')
-    parser.add_argument('--no-remove', action='store_true',
-        help='Do not remove unaccounted files from destination.')
-    parser.add_argument('--no-remove-all-directory-symlinks', action='store_true',
-        help='Do not remove all directory symlinks from destination.')
-    parser.add_argument('--no-remove-empty-directories', action='store_true',
-        help='Do not remove empty directories from destination.')
     parser.add_argument('--no-symlinks', action='store_true',
         help='Do not install symbolic links. Always copy files')
-    parser.add_argument('--track', metavar="PATH",
+    parser.add_argument('--track', metavar="PATH", required=True,
         help='Use installed files tracking information from the given path.')
     parser.add_argument('-D', action=DefinesAction,
         dest='defines', metavar="VAR[=VAL]",
         help='Define a variable to override what is specified in the manifest')
 
     args = parser.parse_args(argv)
 
     start = time.time()
 
     result = process_manifest(args.destdir, args.manifests,
-        track=args.track, remove_unaccounted=not args.no_remove,
-        remove_all_directory_symlinks=not args.no_remove_all_directory_symlinks,
-        remove_empty_directories=not args.no_remove_empty_directories,
+        track=args.track,
         no_symlinks=args.no_symlinks,
         defines=args.defines)
 
     elapsed = time.time() - start
 
     print(COMPLETE.format(
         elapsed=elapsed,
         dest=args.destdir,
--- a/xpcom/xpidl/Makefile.in
+++ b/xpcom/xpidl/Makefile.in
@@ -1,10 +1,10 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 export::
-	$(call py_action,process_install_manifest,$(DIST)/idl $(DEPTH)/_build_manifests/install/dist_idl)
+	$(call py_action,process_install_manifest,--track install-xpidl.track $(DIST)/idl $(DEPTH)/_build_manifests/install/dist_idl)
 	$(call SUBMAKE,xpidl,$(DEPTH)/config/makefiles/xpidl)
 
 clean clobber realclean clobber_all distclean::
 	$(call SUBMAKE,$@,$(DEPTH)/config/makefiles/xpidl)