Backout 312925464acb:f24d5b2801f8 (bug 911375) for breaking the build
authorGregory Szorc <gps@mozilla.com>
Tue, 17 Sep 2013 09:14:58 -0700
changeset 147355 00a1d4006f2b01d2aec95a7881ae05a89c094c80
parent 147354 f24d5b2801f83f7a88b19c9a048392b656546fd0
child 147382 4471e331bb4fb75a4b7ec3bec672e83b2b08e2f5
push id88
push usergszorc@mozilla.com
push dateTue, 17 Sep 2013 16:15:23 +0000
bugs911375
milestone26.0a1
backs out312925464acbda3fb964108c34e7408c2c06a3e3
Backout 312925464acb:f24d5b2801f8 (bug 911375) for breaking the build
Makefile.in
python/mozbuild/mozbuild/action/purge_manifests.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
python/mozbuild/mozpack/manifests.py
python/mozbuild/mozpack/test/test_manifests.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -50,30 +50,18 @@ ifndef MOZ_PROFILE_USE
 # necessary. To manage new directories or add files to the manifests,
 # modify the backend generator.
 #
 # We need to explicitly put backend.RecursiveMakeBackend.built here
 # otherwise the rule in rules.mk doesn't run early enough.
 libs export tools:: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built
 	$(call SUBMAKE,backend.RecursiveMakeBackend.built,js/src,1)
 
-.PHONY: install-manifests
-export:: install-manifests
-
-install_manifests := bin idl include public private sdk
-.PHONY: $(addprefix install-dist-,$(install_manifests))
-install-manifests: $(addprefix install-dist-,$(install_manifests))
-$(addprefix install-dist-,$(install_manifests)): install-dist-%: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built
-	$(call py_action,process_install_manifest,$(DIST)/$* _build_manifests/install/dist_$* js/src/_build_manifests/install/dist_$*)
-
-.PHONY: install-tests
-install-manifests: install-tests
-install-tests: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built
-	$(call py_action,process_install_manifest,_tests _build_manifests/install/tests js/src/_build_manifests/install/tests)
-
+export::
+	$(call py_action,purge_manifests,-d _build_manifests/purge .)
 endif
 
 CLOBBER: $(topsrcdir)/CLOBBER
 	@echo "STOP!  The CLOBBER file has changed."
 	@echo "Please run the build through a sanctioned build wrapper, such as"
 	@echo "'mach build' or client.mk."
 	@exit 1
 
@@ -86,25 +74,25 @@ CLOBBER: $(topsrcdir)/CLOBBER
 
 config.status: $(topsrcdir)/configure
 	@echo "STOP!  configure has changed and needs to be run in this build directory."
 	@echo "Please rerun configure."
 	@echo "To ignore this message, touch 'config.status' in the build directory,"
 	@echo "but your build might not succeed."
 	@exit 1
 
+export::
+	$(RM) -r $(DIST)/sdk
+
 ifdef ENABLE_TESTS
 # Additional makefile targets to call automated test suites
 include $(topsrcdir)/testing/testsuite-targets.mk
 endif
 
 export::
-	$(RM) -r $(DIST)/sdk
-
-export::
 	$(call py_action,process_install_manifest,$(DIST)/include _build_manifests/install/dist_include js/src/_build_manifests/install/dist_include)
 
 default all::
 	$(call BUILDSTATUS,TIERS export compile libs tools)
 
 include $(topsrcdir)/config/rules.mk
 
 distclean::
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/action/purge_manifests.py
@@ -0,0 +1,79 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script is used to purge a directory of unwanted files as defined by
+# a manifest file.
+
+from __future__ import print_function, unicode_literals
+
+import argparse
+import os
+import sys
+import threading
+
+from mozpack.manifests import PurgeManifest
+
+
+def do_purge(purger, dest, state):
+    state['result'] = purger.purge(dest)
+
+
+def process_manifest(topdir, manifest_path):
+    manifest = PurgeManifest(path=manifest_path)
+    purger = manifest.get_purger()
+    full = os.path.join(topdir, manifest.relpath)
+
+    state = dict(
+        relpath=manifest.relpath,
+        result=None,
+    )
+
+    t = threading.Thread(target=do_purge, args=(purger, full, state))
+    state['thread'] = t
+    t.start()
+
+    return state
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Purge a directory of untracked files.')
+
+    parser.add_argument('--directory', '-d',
+        help='Directory containing manifest files. Will process every file '
+            'in directory.')
+    parser.add_argument('topdir',
+        help='Top directory all paths are evaluated from.')
+    parser.add_argument('manifests', nargs='*',
+        help='List of manifest files defining purge operations to perform.')
+
+    args = parser.parse_args()
+
+    states = []
+
+    print('Purging unaccounted files from object directory...')
+
+    # We perform purging using threads for performance reasons. Hopefully
+    # multiple I/O operations will be faster than just 1.
+    paths = []
+    if args.directory:
+        for path in sorted(os.listdir(args.directory)):
+            paths.append(os.path.join(args.directory, path))
+
+    paths.extend(args.manifests)
+
+    for path in paths:
+        states.append(process_manifest(args.topdir, path))
+
+    for state in states:
+        state['thread'].join()
+        print('Deleted %d files and %d directories from %s.' % (
+            state['result'].removed_files_count,
+            state['result'].removed_directories_count,
+            state['relpath']
+        ))
+
+    print('Finished purging.')
+
+    sys.exit(0)
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -7,16 +7,17 @@ from __future__ import unicode_literals
 import errno
 import logging
 import os
 import types
 
 from mozpack.copier import FilePurger
 from mozpack.manifests import (
     InstallManifest,
+    PurgeManifest,
 )
 import mozpack.path as mozpath
 
 from .common import CommonBackend
 from ..frontend.data import (
     ConfigFileSubstitution,
     DirectoryTraversal,
     Exports,
@@ -141,27 +142,29 @@ class RecursiveMakeBackend(CommonBackend
         self.summary.backend_detailed_summary = types.MethodType(detailed,
             self.summary)
 
         self.xpcshell_manifests = []
 
         self.backend_input_files.add(os.path.join(self.environment.topobjdir,
             'config', 'autoconf.mk'))
 
-        self._install_manifests = {
-            k: InstallManifest() for k in [
-                'dist_bin',
-                'dist_idl',
-                'dist_include',
-                'dist_public',
-                'dist_private',
-                'dist_sdk',
-                'tests',
-                'xpidl',
-            ]}
+        self._purge_manifests = dict(
+            dist_bin=PurgeManifest(relpath='dist/bin'),
+            dist_private=PurgeManifest(relpath='dist/private'),
+            dist_public=PurgeManifest(relpath='dist/public'),
+            dist_sdk=PurgeManifest(relpath='dist/sdk'),
+            tests=PurgeManifest(relpath='_tests'),
+            xpidl=PurgeManifest(relpath='config/makefiles/xpidl'),
+        )
+
+        self._install_manifests = dict(
+            dist_idl=InstallManifest(),
+            dist_include=InstallManifest(),
+        )
 
     def _update_from_avoid_write(self, result):
         existed, updated = result
 
         if not existed:
             self.summary.created_count += 1
         elif updated:
             self.summary.updated_count += 1
@@ -345,16 +348,17 @@ class RecursiveMakeBackend(CommonBackend
             mastermanifest.write(
                 '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.\n\n')
             for manifest in self.xpcshell_manifests:
                 mastermanifest.write("[include:%s]\n" % manifest)
             self._update_from_avoid_write(mastermanifest.close())
             self.summary.managed_count += 1
 
         self._write_manifests('install', self._install_manifests)
+        self._write_manifests('purge', self._purge_manifests)
 
     def _process_directory_traversal(self, obj, backend_file):
         """Process a data.DirectoryTraversal instance."""
         fh = backend_file.fh
 
         for tier, dirs in obj.tier_dirs.iteritems():
             fh.write('TIERS += %s\n' % tier)
 
@@ -410,33 +414,31 @@ class RecursiveMakeBackend(CommonBackend
                 raise Exception('File listed in EXPORTS does not exist: %s' % source)
 
         children = exports.get_children()
         for subdir in sorted(children):
             self._process_exports(obj, children[subdir], backend_file,
                 namespace=namespace + subdir)
 
     def _handle_idl_manager(self, manager):
-        build_files = self._install_manifests['xpidl']
+        build_files = self._purge_manifests['xpidl']
 
         for p in ('Makefile', 'backend.mk', '.deps/.mkdir.done',
             'xpt/.mkdir.done'):
-            build_files.add_optional_exists(p)
+            build_files.add(p)
 
         for idl in manager.idls.values():
             self._install_manifests['dist_idl'].add_symlink(idl['source'],
                 idl['basename'])
             self._install_manifests['dist_include'].add_optional_exists('%s.h'
                 % idl['root'])
 
         for module in manager.modules:
-            build_files.add_optional_exists(mozpath.join('xpt',
-                '%s.xpt' % module))
-            build_files.add_optional_exists(mozpath.join('.deps',
-                '%s.pp' % module))
+            build_files.add(mozpath.join('xpt', '%s.xpt' % module))
+            build_files.add(mozpath.join('.deps', '%s.pp' % module))
 
         modules = manager.modules
         xpt_modules = sorted(modules.keys())
         rules = []
 
         for module in xpt_modules:
             deps = sorted(modules[module])
             idl_deps = ['$(dist_idl_dir)/%s.idl' % dep for dep in deps]
@@ -455,17 +457,17 @@ class RecursiveMakeBackend(CommonBackend
                 '\t@echo "$(notdir $@)"',
                 '\t$(idlprocess) $(basename $(notdir $@)) %s' % ' '.join(deps),
                 '',
             ])
 
         # Create dependency for output header so we force regeneration if the
         # header was deleted. This ideally should not be necessary. However,
         # some processes (such as PGO at the time this was implemented) wipe
-        # out dist/include without regard to our install manifests.
+        # out dist/include without regard to our install/purge manifests.
 
         out_path = os.path.join(self.environment.topobjdir, 'config',
             'makefiles', 'xpidl', 'Makefile')
         result = self.environment.create_config_file(out_path, extra=dict(
             xpidl_rules='\n'.join(rules),
             xpidl_modules=' '.join(xpt_modules),
         ))
         self._update_from_avoid_write(result)
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 import os
 
 from mozpack.manifests import (
     InstallManifest,
+    PurgeManifest,
 )
 from mozunit import main
 
 from mozbuild.backend.recursivemake import RecursiveMakeBackend
 from mozbuild.frontend.emitter import TreeMetadataEmitter
 from mozbuild.frontend.reader import BuildReader
 
 from mozbuild.test.backend.common import BackendTester
@@ -265,31 +266,32 @@ class TestRecursiveMakeBackend(BackendTe
         # Assignment[aa], append[cc], conditional[valid]
         expected = ('aa', 'bb', 'cc', 'dd', 'valid_val')
         self.assertEqual(xpclines, ["XPCSHELL_TESTS += %s" % val for val in expected])
 
     def test_xpidl_generation(self):
         """Ensure xpidl files and directories are written out."""
         env = self._consume('xpidl', RecursiveMakeBackend)
 
-        # Install manifests should contain entries.
+        # Purge manifests should contain entries.
+        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
         install_dir = os.path.join(env.topobjdir, '_build_manifests',
             'install')
+        self.assertTrue(os.path.isfile(os.path.join(purge_dir, 'xpidl')))
         self.assertTrue(os.path.isfile(os.path.join(install_dir, 'dist_idl')))
-        self.assertTrue(os.path.isfile(os.path.join(install_dir, 'xpidl')))
+
+        m = PurgeManifest(path=os.path.join(purge_dir, 'xpidl'))
+        self.assertIn('.deps/my_module.pp', m.entries)
+        self.assertIn('xpt/my_module.xpt', m.entries)
 
         m = InstallManifest(path=os.path.join(install_dir, 'dist_idl'))
         self.assertEqual(len(m), 2)
         self.assertIn('bar.idl', m)
         self.assertIn('foo.idl', m)
 
-        m = InstallManifest(path=os.path.join(install_dir, 'xpidl'))
-        self.assertIn('.deps/my_module.pp', m)
-        self.assertIn('xpt/my_module.xpt', m)
-
         m = InstallManifest(path=os.path.join(install_dir, 'dist_include'))
         self.assertIn('foo.h', m)
 
         p = os.path.join(env.topobjdir, 'config/makefiles/xpidl')
         self.assertTrue(os.path.isdir(p))
 
         self.assertTrue(os.path.isfile(os.path.join(p, 'Makefile')))
 
@@ -300,24 +302,45 @@ class TestRecursiveMakeBackend(BackendTe
         manifest_path = os.path.join(env.topobjdir,
             'testing', 'xpcshell', 'xpcshell.ini')
         lines = [l.strip() for l in open(manifest_path, 'rt').readlines()]
         expected = ('aa', 'bb', 'cc', 'dd', 'valid_val')
         self.assertEqual(lines, [
             '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.',
             ''] + ['[include:%s/xpcshell.ini]' % x for x in expected])
 
-    def test_old_install_manifest_deleted(self):
-        # Simulate an install manifest from a previous backend version. Ensure
-        # it is deleted.
+    def test_purge_manifests_written(self):
+        env = self._consume('stub0', RecursiveMakeBackend)
+
+        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
+        self.assertTrue(os.path.exists(purge_dir))
+
+        expected = [
+            'dist_bin',
+            'dist_private',
+            'dist_public',
+            'dist_sdk',
+            'tests',
+        ]
+
+        for e in expected:
+            full = os.path.join(purge_dir, e)
+            self.assertTrue(os.path.exists(full))
+
+        m = PurgeManifest(path=os.path.join(purge_dir, 'dist_bin'))
+        self.assertEqual(m.relpath, 'dist/bin')
+
+    def test_old_purge_manifest_deleted(self):
+        # Simulate a purge manifest from a previous backend version. Ensure it
+        # is deleted.
         env = self._get_environment('stub0')
-        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'install')
+        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
         manifest_path = os.path.join(purge_dir, 'old_manifest')
         os.makedirs(purge_dir)
-        m = InstallManifest()
+        m = PurgeManifest()
         m.write(path=manifest_path)
 
         self.assertTrue(os.path.exists(manifest_path))
         self._consume('stub0', RecursiveMakeBackend, env)
         self.assertFalse(os.path.exists(manifest_path))
 
     def test_install_manifests_written(self):
         env, objs = self._emit('stub0')
--- a/python/mozbuild/mozpack/manifests.py
+++ b/python/mozbuild/mozpack/manifests.py
@@ -29,16 +29,90 @@ def _auto_fileobj(path, fileobj, mode='r
 
     try:
         yield fileobj
     finally:
         if path:
             fileobj.close()
 
 
+class UnreadablePurgeManifest(Exception):
+    """Error for failure when reading content of a serialized PurgeManifest."""
+
+
+class PurgeManifest(object):
+    """Describes actions to be used with a copier.FilePurger instance.
+
+    This class facilitates serialization and deserialization of data used
+    to construct a copier.FilePurger and to perform a purge operation.
+
+    The manifest contains a set of entries (paths that are accounted for and
+    shouldn't be purged) and a relative path. The relative path is optional and
+    can be used to e.g. have several manifest files in a directory be
+    dynamically applied to subdirectories under a common base directory.
+
+    Don't be confused by the name of this class: entries are files that are
+    *not* purged.
+    """
+    def __init__(self, relpath='', path=None, fileobj=None):
+        self.relpath = relpath
+        self.entries = set()
+
+        if not path and not fileobj:
+            return
+
+        with _auto_fileobj(path, fileobj, mode='rt') as fh:
+            self._read_from_fh(fh)
+
+    def __eq__(self, other):
+        if not isinstance(other, PurgeManifest):
+            return False
+
+        return other.relpath == self.relpath and other.entries == self.entries
+
+    def _read_from_fh(self, fileobj):
+        version = fileobj.readline().rstrip()
+        if version != '1':
+            raise UnreadablePurgeManifest('Unknown manifest version: %s' %
+                version)
+
+        self.relpath = fileobj.readline().rstrip()
+
+        for entry in fileobj:
+            self.entries.add(entry.rstrip())
+
+    def add(self, path):
+        return self.entries.add(path)
+
+    def write(self, path=None, fileobj=None):
+        with _auto_fileobj(path, fileobj, 'wt') as fh:
+            fh.write('1\n')
+            fh.write('%s\n' % self.relpath)
+
+            # We write sorted so written output is consistent.
+            for entry in sorted(self.entries):
+                fh.write('%s\n' % entry)
+
+    def get_purger(self, prepend_relpath=False):
+        """Obtain a FilePurger instance from this manifest.
+
+        If :prepend_relpath is truish, the relative path in the manifest will
+        be prepended to paths added to the FilePurger. Otherwise, the raw paths
+        will be used.
+        """
+        p = FilePurger()
+        for entry in self.entries:
+            if prepend_relpath:
+                entry = mozpath.join(self.relpath, entry)
+
+            p.add(entry)
+
+        return p
+
+
 class UnreadableInstallManifest(Exception):
     """Raised when an invalid install manifest is parsed."""
 
 
 class InstallManifest(object):
     """Describes actions to be used with a copier.FileCopier instance.
 
     This class facilitates serialization and deserialization of data used to
--- a/python/mozbuild/mozpack/test/test_manifests.py
+++ b/python/mozbuild/mozpack/test/test_manifests.py
@@ -9,20 +9,53 @@ import os
 import mozunit
 
 from mozpack.copier import (
     FileCopier,
     FileRegistry,
 )
 from mozpack.manifests import (
     InstallManifest,
+    PurgeManifest,
+    UnreadablePurgeManifest,
 )
 from mozpack.test.test_files import TestWithTmpDir
 
 
+class TestPurgeManifest(TestWithTmpDir):
+    def test_construct(self):
+        m = PurgeManifest()
+        self.assertEqual(m.relpath, '')
+        self.assertEqual(len(m.entries), 0)
+
+    def test_serialization(self):
+        m = PurgeManifest(relpath='rel')
+        m.add('foo')
+        m.add('bar')
+        p = self.tmppath('m')
+        m.write(path=p)
+
+        self.assertTrue(os.path.exists(p))
+
+        m2 = PurgeManifest(path=p)
+        self.assertEqual(m.relpath, m2.relpath)
+        self.assertEqual(m.entries, m2.entries)
+        self.assertEqual(m, m2)
+
+    def test_unknown_version(self):
+        p = self.tmppath('bad')
+
+        with open(p, 'wt') as fh:
+            fh.write('2\n')
+            fh.write('not relevant')
+
+        with self.assertRaises(UnreadablePurgeManifest):
+            PurgeManifest(path=p)
+
+
 class TestInstallManifest(TestWithTmpDir):
     def test_construct(self):
         m = InstallManifest()
         self.assertEqual(len(m), 0)
 
     def test_adds(self):
         m = InstallManifest()
         m.add_symlink('s_source', 's_dest')