--- a/python/mozbuild/mozbuild/action/process_install_manifest.py
+++ b/python/mozbuild/mozbuild/action/process_install_manifest.py
@@ -15,16 +15,17 @@ from mozpack.copier import (
 )
 from mozpack.files import (
     BaseFile,
     FileFinder,
 )
 from mozpack.manifests import (
     InstallManifest,
 )
+import mozpack.path as mozpath
 from mozbuild.util import DefinesAction
 
 
 COMPLETE = 'Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; ' \
     'Added/updated {updated}; ' \
     'Removed {rm_files} files and {rm_dirs} directories.'
 
 
@@ -36,19 +37,24 @@ def process_manifest(destdir, paths, tra
         # We use the same format as install manifests for the tracking
         # data.
         manifest = InstallManifest(path=track)
         remove_unaccounted = FileRegistry()
         dummy_file = BaseFile()
 
         finder = FileFinder(destdir, find_dotfiles=True)
         for dest in manifest._dests:
-            for p, f in finder.find(dest):
-                remove_unaccounted.add(p, dummy_file)
-
+            entry = manifest._dests[dest]
+            if entry[0] in (manifest.PATTERN_LINK, manifest.PATTERN_COPY):
+                basefinder = FileFinder(entry[1])
+                for p, f in basefinder.find(entry[2]):
+                    remove_unaccounted.add(mozpath.join(entry[3], p), dummy_file)
+            else:
+                for p, f in finder.find(dest):
+                    remove_unaccounted.add(p, dummy_file)
         remove_empty_directories=True
         remove_all_directory_symlinks=True
 
     else:
         # If tracking is enabled and there is no file, we don't want to
         # be removing anything.
         remove_unaccounted = False
         remove_empty_directories=False