Bug 884587 - Part 1: Perform file removal with purge manifests; r=glandium
☠☠ backed out by ff579fc118dd ☠ ☠
authorGregory Szorc <gps@mozilla.com>
Tue, 25 Jun 2013 11:04:03 -0700
changeset 136401 8d90527c22c60c38ea3492bf67d53dd47455c082
parent 136400 493686bf929575f698629cdf625db8e704172056
child 136402 447ff64adbb1e831302f959b8515bacb384e0089
push id30091
push usergszorc@mozilla.com
push dateTue, 25 Jun 2013 18:04:28 +0000
treeherdermozilla-inbound@447ff64adbb1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs884587
milestone25.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 884587 - Part 1: Perform file removal with purge manifests; r=glandium
Makefile.in
config/purge_directories.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/test/backend/common.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
python/mozbuild/mozpack/manifests.py
python/mozbuild/mozpack/test/test_manifests.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -33,25 +33,39 @@ include $(topsrcdir)/config/config.mk
 GARBAGE_DIRS += dist _javagen _profile _tests staticlib
 DIST_GARBAGE = config.cache config.log config.status* config-defs.h \
    config/autoconf.mk \
    unallmakefiles mozilla-config.h \
    netwerk/necko-config.h xpcom/xpcom-config.h xpcom/xpcom-private.h \
    $(topsrcdir)/.mozconfig.mk $(topsrcdir)/.mozconfig.out
 
 ifndef MOZ_PROFILE_USE
+# One of the first things we do in the build is purge "unknown" files
+# from the object directory. This serves two purposes:
+#
+#   1) Remove files from a previous build no longer accounted for in
+#      this build configuration.
+#
+#   2) Work around poor build system dependencies by forcing some
+#      rebuilds.
+#
+# Ideally #2 does not exist. Our reliance on this aspect should diminish
+# over time.
+#
+# moz.build backend generation simply installs a set of "manifests" into
+# a common directory. Each manifest is responsible for defining files in
+# a specific subdirectory of the object directory. The invoked Python
+# script simply iterates over all the manifests, purging files as
+# necessary. To manage new directories or add files to the manifests,
+# modify the backend generator.
+#
 # We need to explicitly put backend.RecursiveMakeBackend.built here
 # otherwise the rule in rules.mk doesn't run early enough.
 default alldep all:: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built
-	$(RM) -r $(DIST)/sdk
-	$(RM) -r $(DIST)/include
-	$(RM) -r $(DIST)/private
-	$(RM) -r $(DIST)/public
-	$(RM) -r $(DIST)/bin
-	$(RM) -r _tests
+	$(PYTHON) $(topsrcdir)/config/purge_directories.py -d _build_manifests/purge .
 endif
 
 CLOBBER: $(topsrcdir)/CLOBBER
 	@echo "STOP!  The CLOBBER file has changed."
 	@echo "Please run the build through a sanctioned build wrapper, such as"
 	@echo "'mach build' or client.mk."
 	@exit 1
 
new file mode 100644
--- /dev/null
+++ b/config/purge_directories.py
@@ -0,0 +1,77 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script is used to purge a directory of unwanted files as defined by
+# a manifest file.
+
+from __future__ import print_function, unicode_literals
+
+import argparse
+import os
+import sys
+import threading
+
+from mozpack.manifests import PurgeManifest
+
+def do_purge(purger, dest, state):
+    state['result'] = purger.purge(dest)
+
+def process_manifest(topdir, manifest_path):
+    manifest = PurgeManifest.from_path(manifest_path)
+    purger = manifest.get_purger()
+    full = os.path.join(topdir, manifest.relpath)
+
+    state = dict(
+        relpath=manifest.relpath,
+        result=None,
+    )
+
+    t = threading.Thread(target=do_purge, args=(purger, full, state))
+    state['thread'] = t
+    t.start()
+
+    return state
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Purge a directory of untracked files.')
+
+    parser.add_argument('--directory', '-d',
+        help='Directory containing manifest files. Will process every file '
+            'in directory.')
+    parser.add_argument('topdir',
+        help='Top directory all paths are evaluated from.')
+    parser.add_argument('manifests', nargs='*',
+        help='List of manifest files defining purge operations to perform.')
+
+    args = parser.parse_args()
+
+    states = []
+
+    print('Purging unaccounted files from object directory...')
+
+    # We perform purging using threads for performance reasons. Hopefully
+    # multiple I/O operations will be faster than just 1.
+    paths = []
+    if args.directory:
+        for path in sorted(os.listdir(args.directory)):
+            paths.append(os.path.join(args.directory, path))
+
+    paths.extend(args.manifests)
+
+    for path in paths:
+        states.append(process_manifest(args.topdir, path))
+
+    for state in states:
+        state['thread'].join()
+        print('Deleted %d files and %d directories from %s.' % (
+            state['result'].removed_files_count,
+            state['result'].removed_directories_count,
+            state['relpath']
+        ))
+
+    print('Finished purging.')
+
+    sys.exit(0)
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -4,16 +4,19 @@
 
 from __future__ import unicode_literals
 
 import errno
 import logging
 import os
 import types
 
+from mozpack.copier import FilePurger
+from mozpack.manifests import PurgeManifest
+
 from .base import BuildBackend
 from ..frontend.data import (
     ConfigFileSubstitution,
     DirectoryTraversal,
     SandboxDerived,
     VariablePassthru,
     Exports,
     Program,
@@ -122,16 +125,25 @@ class RecursiveMakeBackend(BuildBackend)
         self.summary.backend_detailed_summary = types.MethodType(detailed,
             self.summary)
 
         self.xpcshell_manifests = []
 
         self.backend_input_files.add(os.path.join(self.environment.topobjdir,
             'config', 'autoconf.mk'))
 
+        self._purge_manifests = dict(
+            dist_bin=PurgeManifest(relpath='dist/bin'),
+            dist_include=PurgeManifest(relpath='dist/include'),
+            dist_private=PurgeManifest(relpath='dist/private'),
+            dist_public=PurgeManifest(relpath='dist/public'),
+            dist_sdk=PurgeManifest(relpath='dist/sdk'),
+            tests=PurgeManifest(relpath='_tests'),
+        )
+
     def _update_from_avoid_write(self, result):
         existed, updated = result
 
         if not existed:
             self.summary.created_count += 1
         elif updated:
             self.summary.updated_count += 1
         else:
@@ -247,16 +259,18 @@ class RecursiveMakeBackend(BuildBackend)
                 self.environment.topobjdir, 'testing', 'xpcshell', 'xpcshell.ini'))
             mastermanifest.write(
                 '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.\n\n')
             for manifest in self.xpcshell_manifests:
                 mastermanifest.write("[include:%s]\n" % manifest)
             self._update_from_avoid_write(mastermanifest.close())
             self.summary.managed_count += 1
 
+        self._write_purge_manifests()
+
     def _process_directory_traversal(self, obj, backend_file):
         """Process a data.DirectoryTraversal instance."""
         fh = backend_file.fh
 
         for tier, dirs in obj.tier_dirs.iteritems():
             fh.write('TIERS += %s\n' % tier)
 
             if dirs:
@@ -318,8 +332,32 @@ class RecursiveMakeBackend(BuildBackend)
         backend_file.write('PROGRAM = %s\n' % program)
 
     def _process_xpcshell_manifests(self, obj, backend_file, namespace=""):
         manifest = obj.xpcshell_manifests
         backend_file.write('XPCSHELL_TESTS += %s\n' % os.path.dirname(manifest))
         if obj.relativedir != '':
             manifest = '%s/%s' % (obj.relativedir, manifest)
         self.xpcshell_manifests.append(manifest)
+
+    def _write_purge_manifests(self):
+        # We write out a "manifest" file for each directory that is to be
+        # purged.
+        #
+        # Ideally we have as few manifests as possible - ideally only 1. This
+        # will likely require all build metadata to be in emitted objects.
+        # We're not quite there yet, so we maintain multiple manifests.
+        man_dir = os.path.join(self.environment.topobjdir, '_build_manifests',
+            'purge')
+
+        # We have a purger for the manifests themselves to ensure we don't over
+        # purge if we delete a purge manifest.
+        purger = FilePurger()
+
+        for k, manifest in self._purge_manifests.items():
+            purger.add(k)
+            full = os.path.join(man_dir, k)
+
+            fh = FileAvoidWrite(os.path.join(man_dir, k))
+            manifest.write_fileobj(fh)
+            self._update_from_avoid_write(fh.close())
+
+        purger.purge(man_dir)
--- a/python/mozbuild/mozbuild/test/backend/common.py
+++ b/python/mozbuild/mozbuild/test/backend/common.py
@@ -84,25 +84,27 @@ class BackendTester(unittest.TestCase):
 
         objdir = mkdtemp()
         self.addCleanup(rmtree, objdir)
 
         srcdir = os.path.join(test_data_path, name)
         config['substs'].append(('top_srcdir', srcdir))
         return ConfigEnvironment(srcdir, objdir, **config)
 
-    def _emit(self, name):
-        env = self._get_environment(name)
+    def _emit(self, name, env=None):
+        if not env:
+            env = self._get_environment(name)
+
         reader = BuildReader(env)
         emitter = TreeMetadataEmitter(env)
 
         return env, emitter.emit(reader.read_topsrcdir())
 
-    def _consume(self, name, cls):
-        env, objs = self._emit(name)
+    def _consume(self, name, cls, env=None):
+        env, objs = self._emit(name, env=env)
         backend = cls(env)
         backend.consume(objs)
 
         return env
 
     def _tree_paths(self, topdir, filename):
         for dirpath, dirnames, filenames in os.walk(topdir):
             for f in filenames:
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -2,16 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 import os
 import time
 
+from mozpack.manifests import PurgeManifest
 from mozunit import main
 
 from mozbuild.backend.configenvironment import ConfigEnvironment
 from mozbuild.backend.recursivemake import RecursiveMakeBackend
 from mozbuild.frontend.emitter import TreeMetadataEmitter
 from mozbuild.frontend.reader import BuildReader
 
 from mozbuild.test.backend.common import BackendTester
@@ -252,10 +253,47 @@ class TestRecursiveMakeBackend(BackendTe
         manifest_path = os.path.join(env.topobjdir,
             'testing', 'xpcshell', 'xpcshell.ini')
         lines = [l.strip() for l in open(manifest_path, 'rt').readlines()]
         expected = ('aa', 'bb', 'cc', 'dd', 'valid_val')
         self.assertEqual(lines, [
             '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.',
             ''] + ['[include:%s/xpcshell.ini]' % x for x in expected])
 
+    def test_purge_manifests_written(self):
+        env = self._consume('stub0', RecursiveMakeBackend)
+
+        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
+        self.assertTrue(os.path.exists(purge_dir))
+
+        expected = [
+            'dist_bin',
+            'dist_include',
+            'dist_private',
+            'dist_public',
+            'dist_sdk',
+            'tests',
+        ]
+
+        for e in expected:
+            full = os.path.join(purge_dir, e)
+            self.assertTrue(os.path.exists(full))
+
+        m = PurgeManifest.from_path(os.path.join(purge_dir, 'dist_bin'))
+        self.assertEqual(m.relpath, 'dist/bin')
+
+    def test_old_purge_manifest_deleted(self):
+        # Simulate a purge manifest from a previous backend version. Ensure it
+        # is deleted.
+        env = self._get_environment('stub0')
+        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
+        manifest_path = os.path.join(purge_dir, 'old_manifest')
+        os.makedirs(purge_dir)
+        m = PurgeManifest()
+        m.write_file(manifest_path)
+
+        self.assertTrue(os.path.exists(manifest_path))
+        self._consume('stub0', RecursiveMakeBackend, env)
+        self.assertFalse(os.path.exists(manifest_path))
+
+
 if __name__ == '__main__':
     main()
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozpack/manifests.py
@@ -0,0 +1,89 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+from .copier import FilePurger
+import mozpack.path as mozpath
+
+
+class UnreadablePurgeManifest(Exception):
+    """Error for failure when reading content of a serialized PurgeManifest."""
+
+
+class PurgeManifest(object):
+    """Describes actions to be used with a copier.FilePurger instance.
+
+    This class facilitates serialization and deserialization of data used
+    to construct a copier.FilePurger and to perform a purge operation.
+
+    The manifest contains a set of entries (paths that are accounted for and
+    shouldn't be purged) and a relative path. The relative path is optional and
+    can be used to e.g. have several manifest files in a directory be
+    dynamically applied to subdirectories under a common base directory.
+
+    Don't be confused by the name of this class: entries are files that are
+    *not* purged.
+    """
+    def __init__(self, relpath=''):
+        self.relpath = relpath
+        self.entries = set()
+
+    def __eq__(self, other):
+        if not isinstance(other, PurgeManifest):
+            return False
+
+        return other.relpath == self.relpath and other.entries == self.entries
+
+    @staticmethod
+    def from_path(path):
+        with open(path, 'rt') as fh:
+            return PurgeManifest.from_fileobj(fh)
+
+    @staticmethod
+    def from_fileobj(fh):
+        m = PurgeManifest()
+
+        version = fh.readline().rstrip()
+        if version != '1':
+            raise UnreadablePurgeManifest('Unknown manifest version: ' %
+                version)
+
+        m.relpath = fh.readline().rstrip()
+
+        for entry in fh:
+            m.entries.add(entry.rstrip())
+
+        return m
+
+    def add(self, path):
+        return self.entries.add(path)
+
+    def write_file(self, path):
+        with open(path, 'wt') as fh:
+            return self.write_fileobj(fh)
+
+    def write_fileobj(self, fh):
+        fh.write('1\n')
+        fh.write('%s\n' % self.relpath)
+
+        # We write sorted so written output is consistent.
+        for entry in sorted(self.entries):
+            fh.write('%s\n' % entry)
+
+    def get_purger(self, prepend_relpath=False):
+        """Obtain a FilePurger instance from this manifest.
+
+        If :prepend_relpath is truish, the relative path in the manifest will
+        be prepended to paths added to the FilePurger. Otherwise, the raw paths
+        will be used.
+        """
+        p = FilePurger()
+        for entry in self.entries:
+            if prepend_relpath:
+                entry = mozpath.join(self.relpath, entry)
+
+            p.add(entry)
+
+        return p
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozpack/test/test_manifests.py
@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+import unittest
+
+import mozunit
+
+from mozpack.manifests import (
+    PurgeManifest,
+    UnreadablePurgeManifest,
+)
+from mozpack.test.test_files import TestWithTmpDir
+
+
+class TestPurgeManifest(TestWithTmpDir):
+    def test_construct(self):
+        m = PurgeManifest()
+        self.assertEqual(m.relpath, '')
+        self.assertEqual(len(m.entries), 0)
+
+    def test_serialization(self):
+        m = PurgeManifest(relpath='rel')
+        m.add('foo')
+        m.add('bar')
+        p = self.tmppath('m')
+        m.write_file(p)
+
+        self.assertTrue(os.path.exists(p))
+
+        m2 = PurgeManifest.from_path(p)
+        self.assertEqual(m.relpath, m2.relpath)
+        self.assertEqual(m.entries, m2.entries)
+        self.assertEqual(m, m2)
+
+    def test_unknown_version(self):
+        p = self.tmppath('bad')
+
+        with open(p, 'wt') as fh:
+            fh.write('2\n')
+            fh.write('not relevant')
+
+        with self.assertRaises(UnreadablePurgeManifest):
+            PurgeManifest.from_path(p)
+