Bug 884587 - Part 1: Perform file removal with purge manifests; r=glandium
☠☠ backed out by 301c184ae8e7 ☠ ☠
authorGregory Szorc <gps@mozilla.com>
Tue, 25 Jun 2013 11:04:03 -0700
changeset 144021 796961a384b47798de261e2c9c6b331f39144f1c
parent 144020 ca137844253ff6ec4b6b88812fadb29990a1ea54
child 144022 0244a34bc4190998441c143140451598fe941c66
push idunknown
push userunknown
push dateunknown
reviewersglandium
bugs884587
milestone25.0a1
Bug 884587 - Part 1: Perform file removal with purge manifests; r=glandium
Makefile.in
config/purge_directories.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/test/backend/common.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
python/mozbuild/mozpack/manifests.py
python/mozbuild/mozpack/test/test_manifests.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -33,25 +33,39 @@ include $(topsrcdir)/config/config.mk
 GARBAGE_DIRS += dist _javagen _profile _tests staticlib
 DIST_GARBAGE = config.cache config.log config.status* config-defs.h \
    config/autoconf.mk \
    unallmakefiles mozilla-config.h \
    netwerk/necko-config.h xpcom/xpcom-config.h xpcom/xpcom-private.h \
    $(topsrcdir)/.mozconfig.mk $(topsrcdir)/.mozconfig.out
 
 ifndef MOZ_PROFILE_USE
+# One of the first things we do in the build is purge "unknown" files
+# from the object directory. This serves two purposes:
+#
+#   1) Remove files from a previous build no longer accounted for in
+#      this build configuration.
+#
+#   2) Work around poor build system dependencies by forcing some
+#      rebuilds.
+#
+# Ideally #2 does not exist. Our reliance on this aspect should diminish
+# over time.
+#
+# moz.build backend generation simply installs a set of "manifests" into
+# a common directory. Each manifest is responsible for defining files in
+# a specific subdirectory of the object directory. The invoked Python
+# script simply iterates over all the manifests, purging files as
+# necessary. To manage new directories or add files to the manifests,
+# modify the backend generator.
+#
 # We need to explicitly put backend.RecursiveMakeBackend.built here
 # otherwise the rule in rules.mk doesn't run early enough.
 default alldep all:: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built
-	$(RM) -r $(DIST)/sdk
-	$(RM) -r $(DIST)/include
-	$(RM) -r $(DIST)/private
-	$(RM) -r $(DIST)/public
-	$(RM) -r $(DIST)/bin
-	$(RM) -r _tests
+	$(PYTHON) $(topsrcdir)/config/purge_directories.py -d _build_manifests/purge .
 endif
 
 CLOBBER: $(topsrcdir)/CLOBBER
 	@echo "STOP!  The CLOBBER file has changed."
 	@echo "Please run the build through a sanctioned build wrapper, such as"
 	@echo "'mach build' or client.mk."
 	@exit 1
 
new file mode 100644
--- /dev/null
+++ b/config/purge_directories.py
@@ -0,0 +1,77 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script is used to purge a directory of unwanted files as defined by
+# a manifest file.
+
+from __future__ import print_function, unicode_literals
+
+import argparse
+import os
+import sys
+import threading
+
+from mozpack.manifests import PurgeManifest
+
+def do_purge(purger, dest, state):
+    state['result'] = purger.purge(dest)
+
+def process_manifest(topdir, manifest_path):
+    manifest = PurgeManifest.from_path(manifest_path)
+    purger = manifest.get_purger()
+    full = os.path.join(topdir, manifest.relpath)
+
+    state = dict(
+        relpath=manifest.relpath,
+        result=None,
+    )
+
+    t = threading.Thread(target=do_purge, args=(purger, full, state))
+    state['thread'] = t
+    t.start()
+
+    return state
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Purge a directory of untracked files.')
+
+    parser.add_argument('--directory', '-d',
+        help='Directory containing manifest files. Will process every file '
+            'in directory.')
+    parser.add_argument('topdir',
+        help='Top directory all paths are evaluated from.')
+    parser.add_argument('manifests', nargs='*',
+        help='List of manifest files defining purge operations to perform.')
+
+    args = parser.parse_args()
+
+    states = []
+
+    print('Purging unaccounted files from object directory...')
+
+    # We perform purging using threads for performance reasons. Hopefully
+    # multiple I/O operations will be faster than just 1.
+    paths = []
+    if args.directory:
+        for path in sorted(os.listdir(args.directory)):
+            paths.append(os.path.join(args.directory, path))
+
+    paths.extend(args.manifests)
+
+    for path in paths:
+        states.append(process_manifest(args.topdir, path))
+
+    for state in states:
+        state['thread'].join()
+        print('Deleted %d files and %d directories from %s.' % (
+            state['result'].removed_files_count,
+            state['result'].removed_directories_count,
+            state['relpath']
+        ))
+
+    print('Finished purging.')
+
+    sys.exit(0)
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -4,16 +4,19 @@
 
 from __future__ import unicode_literals
 
 import errno
 import logging
 import os
 import types
 
+from mozpack.copier import FilePurger
+from mozpack.manifests import PurgeManifest
+
 from .base import BuildBackend
 from ..frontend.data import (
     ConfigFileSubstitution,
     DirectoryTraversal,
     SandboxDerived,
     VariablePassthru,
     Exports,
     Program,
@@ -122,16 +125,25 @@ class RecursiveMakeBackend(BuildBackend)
         self.summary.backend_detailed_summary = types.MethodType(detailed,
             self.summary)
 
         self.xpcshell_manifests = []
 
         self.backend_input_files.add(os.path.join(self.environment.topobjdir,
             'config', 'autoconf.mk'))
 
+        self._purge_manifests = dict(
+            dist_bin=PurgeManifest(relpath='dist/bin'),
+            dist_include=PurgeManifest(relpath='dist/include'),
+            dist_private=PurgeManifest(relpath='dist/private'),
+            dist_public=PurgeManifest(relpath='dist/public'),
+            dist_sdk=PurgeManifest(relpath='dist/sdk'),
+            tests=PurgeManifest(relpath='_tests'),
+        )
+
     def _update_from_avoid_write(self, result):
         existed, updated = result
 
         if not existed:
             self.summary.created_count += 1
         elif updated:
             self.summary.updated_count += 1
         else:
@@ -247,16 +259,18 @@ class RecursiveMakeBackend(BuildBackend)
                 self.environment.topobjdir, 'testing', 'xpcshell', 'xpcshell.ini'))
             mastermanifest.write(
                 '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.\n\n')
             for manifest in self.xpcshell_manifests:
                 mastermanifest.write("[include:%s]\n" % manifest)
             self._update_from_avoid_write(mastermanifest.close())
             self.summary.managed_count += 1
 
+        self._write_purge_manifests()
+
     def _process_directory_traversal(self, obj, backend_file):
         """Process a data.DirectoryTraversal instance."""
         fh = backend_file.fh
 
         for tier, dirs in obj.tier_dirs.iteritems():
             fh.write('TIERS += %s\n' % tier)
 
             if dirs:
@@ -318,8 +332,32 @@ class RecursiveMakeBackend(BuildBackend)
         backend_file.write('PROGRAM = %s\n' % program)
 
     def _process_xpcshell_manifests(self, obj, backend_file, namespace=""):
         manifest = obj.xpcshell_manifests
         backend_file.write('XPCSHELL_TESTS += %s\n' % os.path.dirname(manifest))
         if obj.relativedir != '':
             manifest = '%s/%s' % (obj.relativedir, manifest)
         self.xpcshell_manifests.append(manifest)
+
+    def _write_purge_manifests(self):
+        # We write out a "manifest" file for each directory that is to be
+        # purged.
+        #
+        # Ideally we have as few manifests as possible - ideally only 1. This
+        # will likely require all build metadata to be in emitted objects.
+        # We're not quite there yet, so we maintain multiple manifests.
+        man_dir = os.path.join(self.environment.topobjdir, '_build_manifests',
+            'purge')
+
+        # We have a purger for the manifests themselves to ensure we don't over
+        # purge if we delete a purge manifest.
+        purger = FilePurger()
+
+        for k, manifest in self._purge_manifests.items():
+            purger.add(k)
+            full = os.path.join(man_dir, k)
+
+            fh = FileAvoidWrite(os.path.join(man_dir, k))
+            manifest.write_fileobj(fh)
+            self._update_from_avoid_write(fh.close())
+
+        purger.purge(man_dir)
--- a/python/mozbuild/mozbuild/test/backend/common.py
+++ b/python/mozbuild/mozbuild/test/backend/common.py
@@ -84,25 +84,27 @@ class BackendTester(unittest.TestCase):
 
         objdir = mkdtemp()
         self.addCleanup(rmtree, objdir)
 
         srcdir = os.path.join(test_data_path, name)
         config['substs'].append(('top_srcdir', srcdir))
         return ConfigEnvironment(srcdir, objdir, **config)
 
-    def _emit(self, name):
-        env = self._get_environment(name)
+    def _emit(self, name, env=None):
+        if not env:
+            env = self._get_environment(name)
+
         reader = BuildReader(env)
         emitter = TreeMetadataEmitter(env)
 
         return env, emitter.emit(reader.read_topsrcdir())
 
-    def _consume(self, name, cls):
-        env, objs = self._emit(name)
+    def _consume(self, name, cls, env=None):
+        env, objs = self._emit(name, env=env)
         backend = cls(env)
         backend.consume(objs)
 
         return env
 
     def _tree_paths(self, topdir, filename):
         for dirpath, dirnames, filenames in os.walk(topdir):
             for f in filenames:
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -2,16 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 import os
 import time
 
+from mozpack.manifests import PurgeManifest
 from mozunit import main
 
 from mozbuild.backend.configenvironment import ConfigEnvironment
 from mozbuild.backend.recursivemake import RecursiveMakeBackend
 from mozbuild.frontend.emitter import TreeMetadataEmitter
 from mozbuild.frontend.reader import BuildReader
 
 from mozbuild.test.backend.common import BackendTester
@@ -252,10 +253,47 @@ class TestRecursiveMakeBackend(BackendTe
         manifest_path = os.path.join(env.topobjdir,
             'testing', 'xpcshell', 'xpcshell.ini')
         lines = [l.strip() for l in open(manifest_path, 'rt').readlines()]
         expected = ('aa', 'bb', 'cc', 'dd', 'valid_val')
         self.assertEqual(lines, [
             '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.',
             ''] + ['[include:%s/xpcshell.ini]' % x for x in expected])
 
+    def test_purge_manifests_written(self):
+        env = self._consume('stub0', RecursiveMakeBackend)
+
+        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
+        self.assertTrue(os.path.exists(purge_dir))
+
+        expected = [
+            'dist_bin',
+            'dist_include',
+            'dist_private',
+            'dist_public',
+            'dist_sdk',
+            'tests',
+        ]
+
+        for e in expected:
+            full = os.path.join(purge_dir, e)
+            self.assertTrue(os.path.exists(full))
+
+        m = PurgeManifest.from_path(os.path.join(purge_dir, 'dist_bin'))
+        self.assertEqual(m.relpath, 'dist/bin')
+
+    def test_old_purge_manifest_deleted(self):
+        # Simulate a purge manifest from a previous backend version. Ensure it
+        # is deleted.
+        env = self._get_environment('stub0')
+        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
+        manifest_path = os.path.join(purge_dir, 'old_manifest')
+        os.makedirs(purge_dir)
+        m = PurgeManifest()
+        m.write_file(manifest_path)
+
+        self.assertTrue(os.path.exists(manifest_path))
+        self._consume('stub0', RecursiveMakeBackend, env)
+        self.assertFalse(os.path.exists(manifest_path))
+
+
 if __name__ == '__main__':
     main()
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozpack/manifests.py
@@ -0,0 +1,89 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+from .copier import FilePurger
+import mozpack.path as mozpath
+
+
+class UnreadablePurgeManifest(Exception):
+    """Error for failure when reading content of a serialized PurgeManifest."""
+
+
+class PurgeManifest(object):
+    """Describes actions to be used with a copier.FilePurger instance.
+
+    This class facilitates serialization and deserialization of data used
+    to construct a copier.FilePurger and to perform a purge operation.
+
+    The manifest contains a set of entries (paths that are accounted for and
+    shouldn't be purged) and a relative path. The relative path is optional and
+    can be used to e.g. have several manifest files in a directory be
+    dynamically applied to subdirectories under a common base directory.
+
+    Don't be confused by the name of this class: entries are files that are
+    *not* purged.
+    """
+    def __init__(self, relpath=''):
+        self.relpath = relpath
+        self.entries = set()
+
+    def __eq__(self, other):
+        if not isinstance(other, PurgeManifest):
+            return False
+
+        return other.relpath == self.relpath and other.entries == self.entries
+
+    @staticmethod
+    def from_path(path):
+        with open(path, 'rt') as fh:
+            return PurgeManifest.from_fileobj(fh)
+
+    @staticmethod
+    def from_fileobj(fh):
+        m = PurgeManifest()
+
+        version = fh.readline().rstrip()
+        if version != '1':
+            raise UnreadablePurgeManifest('Unknown manifest version: ' %
+                version)
+
+        m.relpath = fh.readline().rstrip()
+
+        for entry in fh:
+            m.entries.add(entry.rstrip())
+
+        return m
+
+    def add(self, path):
+        return self.entries.add(path)
+
+    def write_file(self, path):
+        with open(path, 'wt') as fh:
+            return self.write_fileobj(fh)
+
+    def write_fileobj(self, fh):
+        fh.write('1\n')
+        fh.write('%s\n' % self.relpath)
+
+        # We write sorted so written output is consistent.
+        for entry in sorted(self.entries):
+            fh.write('%s\n' % entry)
+
+    def get_purger(self, prepend_relpath=False):
+        """Obtain a FilePurger instance from this manifest.
+
+        If :prepend_relpath is truish, the relative path in the manifest will
+        be prepended to paths added to the FilePurger. Otherwise, the raw paths
+        will be used.
+        """
+        p = FilePurger()
+        for entry in self.entries:
+            if prepend_relpath:
+                entry = mozpath.join(self.relpath, entry)
+
+            p.add(entry)
+
+        return p
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozpack/test/test_manifests.py
@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+import unittest
+
+import mozunit
+
+from mozpack.manifests import (
+    PurgeManifest,
+    UnreadablePurgeManifest,
+)
+from mozpack.test.test_files import TestWithTmpDir
+
+
+class TestPurgeManifest(TestWithTmpDir):
+    def test_construct(self):
+        m = PurgeManifest()
+        self.assertEqual(m.relpath, '')
+        self.assertEqual(len(m.entries), 0)
+
+    def test_serialization(self):
+        m = PurgeManifest(relpath='rel')
+        m.add('foo')
+        m.add('bar')
+        p = self.tmppath('m')
+        m.write_file(p)
+
+        self.assertTrue(os.path.exists(p))
+
+        m2 = PurgeManifest.from_path(p)
+        self.assertEqual(m.relpath, m2.relpath)
+        self.assertEqual(m.entries, m2.entries)
+        self.assertEqual(m, m2)
+
+    def test_unknown_version(self):
+        p = self.tmppath('bad')
+
+        with open(p, 'wt') as fh:
+            fh.write('2\n')
+            fh.write('not relevant')
+
+        with self.assertRaises(UnreadablePurgeManifest):
+            PurgeManifest.from_path(p)
+