Backout 312925464acb:f24d5b2801f8 (bug 911375) for breaking the build
authorGregory Szorc <gps@mozilla.com>
Tue, 17 Sep 2013 09:14:58 -0700
changeset 147515 00a1d4006f2b01d2aec95a7881ae05a89c094c80
parent 147514 f24d5b2801f83f7a88b19c9a048392b656546fd0
child 147516 4471e331bb4fb75a4b7ec3bec672e83b2b08e2f5
push id33907
push userryanvm@gmail.com
push dateTue, 17 Sep 2013 20:58:38 +0000
treeherdermozilla-inbound@eb501139f68a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs911375
milestone26.0a1
backs out312925464acbda3fb964108c34e7408c2c06a3e3
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backout 312925464acb:f24d5b2801f8 (bug 911375) for breaking the build
Makefile.in
python/mozbuild/mozbuild/action/purge_manifests.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
python/mozbuild/mozpack/manifests.py
python/mozbuild/mozpack/test/test_manifests.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -50,30 +50,18 @@ ifndef MOZ_PROFILE_USE
 # necessary. To manage new directories or add files to the manifests,
 # modify the backend generator.
 #
 # We need to explicitly put backend.RecursiveMakeBackend.built here
 # otherwise the rule in rules.mk doesn't run early enough.
 libs export tools:: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built
 	$(call SUBMAKE,backend.RecursiveMakeBackend.built,js/src,1)
 
-.PHONY: install-manifests
-export:: install-manifests
-
-install_manifests := bin idl include public private sdk
-.PHONY: $(addprefix install-dist-,$(install_manifests))
-install-manifests: $(addprefix install-dist-,$(install_manifests))
-$(addprefix install-dist-,$(install_manifests)): install-dist-%: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built
-	$(call py_action,process_install_manifest,$(DIST)/$* _build_manifests/install/dist_$* js/src/_build_manifests/install/dist_$*)
-
-.PHONY: install-tests
-install-manifests: install-tests
-install-tests: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built
-	$(call py_action,process_install_manifest,_tests _build_manifests/install/tests js/src/_build_manifests/install/tests)
-
+export::
+	$(call py_action,purge_manifests,-d _build_manifests/purge .)
 endif
 
 CLOBBER: $(topsrcdir)/CLOBBER
 	@echo "STOP!  The CLOBBER file has changed."
 	@echo "Please run the build through a sanctioned build wrapper, such as"
 	@echo "'mach build' or client.mk."
 	@exit 1
 
@@ -86,25 +74,25 @@ CLOBBER: $(topsrcdir)/CLOBBER
 
 config.status: $(topsrcdir)/configure
 	@echo "STOP!  configure has changed and needs to be run in this build directory."
 	@echo "Please rerun configure."
 	@echo "To ignore this message, touch 'config.status' in the build directory,"
 	@echo "but your build might not succeed."
 	@exit 1
 
+export::
+	$(RM) -r $(DIST)/sdk
+
 ifdef ENABLE_TESTS
 # Additional makefile targets to call automated test suites
 include $(topsrcdir)/testing/testsuite-targets.mk
 endif
 
 export::
-	$(RM) -r $(DIST)/sdk
-
-export::
 	$(call py_action,process_install_manifest,$(DIST)/include _build_manifests/install/dist_include js/src/_build_manifests/install/dist_include)
 
 default all::
 	$(call BUILDSTATUS,TIERS export compile libs tools)
 
 include $(topsrcdir)/config/rules.mk
 
 distclean::
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/action/purge_manifests.py
@@ -0,0 +1,79 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script is used to purge a directory of unwanted files as defined by
+# a manifest file.
+
+from __future__ import print_function, unicode_literals
+
+import argparse
+import os
+import sys
+import threading
+
+from mozpack.manifests import PurgeManifest
+
+
+def do_purge(purger, dest, state):
+    state['result'] = purger.purge(dest)
+
+
+def process_manifest(topdir, manifest_path):
+    manifest = PurgeManifest(path=manifest_path)
+    purger = manifest.get_purger()
+    full = os.path.join(topdir, manifest.relpath)
+
+    state = dict(
+        relpath=manifest.relpath,
+        result=None,
+    )
+
+    t = threading.Thread(target=do_purge, args=(purger, full, state))
+    state['thread'] = t
+    t.start()
+
+    return state
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Purge a directory of untracked files.')
+
+    parser.add_argument('--directory', '-d',
+        help='Directory containing manifest files. Will process every file '
+            'in directory.')
+    parser.add_argument('topdir',
+        help='Top directory all paths are evaluated from.')
+    parser.add_argument('manifests', nargs='*',
+        help='List of manifest files defining purge operations to perform.')
+
+    args = parser.parse_args()
+
+    states = []
+
+    print('Purging unaccounted files from object directory...')
+
+    # We perform purging using threads for performance reasons. Hopefully
+    # multiple I/O operations will be faster than just 1.
+    paths = []
+    if args.directory:
+        for path in sorted(os.listdir(args.directory)):
+            paths.append(os.path.join(args.directory, path))
+
+    paths.extend(args.manifests)
+
+    for path in paths:
+        states.append(process_manifest(args.topdir, path))
+
+    for state in states:
+        state['thread'].join()
+        print('Deleted %d files and %d directories from %s.' % (
+            state['result'].removed_files_count,
+            state['result'].removed_directories_count,
+            state['relpath']
+        ))
+
+    print('Finished purging.')
+
+    sys.exit(0)
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -7,16 +7,17 @@ from __future__ import unicode_literals
 import errno
 import logging
 import os
 import types
 
 from mozpack.copier import FilePurger
 from mozpack.manifests import (
     InstallManifest,
+    PurgeManifest,
 )
 import mozpack.path as mozpath
 
 from .common import CommonBackend
 from ..frontend.data import (
     ConfigFileSubstitution,
     DirectoryTraversal,
     Exports,
@@ -141,27 +142,29 @@ class RecursiveMakeBackend(CommonBackend
         self.summary.backend_detailed_summary = types.MethodType(detailed,
             self.summary)
 
         self.xpcshell_manifests = []
 
         self.backend_input_files.add(os.path.join(self.environment.topobjdir,
             'config', 'autoconf.mk'))
 
-        self._install_manifests = {
-            k: InstallManifest() for k in [
-                'dist_bin',
-                'dist_idl',
-                'dist_include',
-                'dist_public',
-                'dist_private',
-                'dist_sdk',
-                'tests',
-                'xpidl',
-            ]}
+        self._purge_manifests = dict(
+            dist_bin=PurgeManifest(relpath='dist/bin'),
+            dist_private=PurgeManifest(relpath='dist/private'),
+            dist_public=PurgeManifest(relpath='dist/public'),
+            dist_sdk=PurgeManifest(relpath='dist/sdk'),
+            tests=PurgeManifest(relpath='_tests'),
+            xpidl=PurgeManifest(relpath='config/makefiles/xpidl'),
+        )
+
+        self._install_manifests = dict(
+            dist_idl=InstallManifest(),
+            dist_include=InstallManifest(),
+        )
 
     def _update_from_avoid_write(self, result):
         existed, updated = result
 
         if not existed:
             self.summary.created_count += 1
         elif updated:
             self.summary.updated_count += 1
@@ -345,16 +348,17 @@ class RecursiveMakeBackend(CommonBackend
             mastermanifest.write(
                 '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.\n\n')
             for manifest in self.xpcshell_manifests:
                 mastermanifest.write("[include:%s]\n" % manifest)
             self._update_from_avoid_write(mastermanifest.close())
             self.summary.managed_count += 1
 
         self._write_manifests('install', self._install_manifests)
+        self._write_manifests('purge', self._purge_manifests)
 
     def _process_directory_traversal(self, obj, backend_file):
         """Process a data.DirectoryTraversal instance."""
         fh = backend_file.fh
 
         for tier, dirs in obj.tier_dirs.iteritems():
             fh.write('TIERS += %s\n' % tier)
 
@@ -410,33 +414,31 @@ class RecursiveMakeBackend(CommonBackend
                 raise Exception('File listed in EXPORTS does not exist: %s' % source)
 
         children = exports.get_children()
         for subdir in sorted(children):
             self._process_exports(obj, children[subdir], backend_file,
                 namespace=namespace + subdir)
 
     def _handle_idl_manager(self, manager):
-        build_files = self._install_manifests['xpidl']
+        build_files = self._purge_manifests['xpidl']
 
         for p in ('Makefile', 'backend.mk', '.deps/.mkdir.done',
             'xpt/.mkdir.done'):
-            build_files.add_optional_exists(p)
+            build_files.add(p)
 
         for idl in manager.idls.values():
             self._install_manifests['dist_idl'].add_symlink(idl['source'],
                 idl['basename'])
             self._install_manifests['dist_include'].add_optional_exists('%s.h'
                 % idl['root'])
 
         for module in manager.modules:
-            build_files.add_optional_exists(mozpath.join('xpt',
-                '%s.xpt' % module))
-            build_files.add_optional_exists(mozpath.join('.deps',
-                '%s.pp' % module))
+            build_files.add(mozpath.join('xpt', '%s.xpt' % module))
+            build_files.add(mozpath.join('.deps', '%s.pp' % module))
 
         modules = manager.modules
         xpt_modules = sorted(modules.keys())
         rules = []
 
         for module in xpt_modules:
             deps = sorted(modules[module])
             idl_deps = ['$(dist_idl_dir)/%s.idl' % dep for dep in deps]
@@ -455,17 +457,17 @@ class RecursiveMakeBackend(CommonBackend
                 '\t@echo "$(notdir $@)"',
                 '\t$(idlprocess) $(basename $(notdir $@)) %s' % ' '.join(deps),
                 '',
             ])
 
         # Create dependency for output header so we force regeneration if the
         # header was deleted. This ideally should not be necessary. However,
         # some processes (such as PGO at the time this was implemented) wipe
-        # out dist/include without regard to our install manifests.
+        # out dist/include without regard to our install/purge manifests.
 
         out_path = os.path.join(self.environment.topobjdir, 'config',
             'makefiles', 'xpidl', 'Makefile')
         result = self.environment.create_config_file(out_path, extra=dict(
             xpidl_rules='\n'.join(rules),
             xpidl_modules=' '.join(xpt_modules),
         ))
         self._update_from_avoid_write(result)
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 import os
 
 from mozpack.manifests import (
     InstallManifest,
+    PurgeManifest,
 )
 from mozunit import main
 
 from mozbuild.backend.recursivemake import RecursiveMakeBackend
 from mozbuild.frontend.emitter import TreeMetadataEmitter
 from mozbuild.frontend.reader import BuildReader
 
 from mozbuild.test.backend.common import BackendTester
@@ -265,31 +266,32 @@ class TestRecursiveMakeBackend(BackendTe
         # Assignment[aa], append[cc], conditional[valid]
         expected = ('aa', 'bb', 'cc', 'dd', 'valid_val')
         self.assertEqual(xpclines, ["XPCSHELL_TESTS += %s" % val for val in expected])
 
     def test_xpidl_generation(self):
         """Ensure xpidl files and directories are written out."""
         env = self._consume('xpidl', RecursiveMakeBackend)
 
-        # Install manifests should contain entries.
+        # Purge manifests should contain entries.
+        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
         install_dir = os.path.join(env.topobjdir, '_build_manifests',
             'install')
+        self.assertTrue(os.path.isfile(os.path.join(purge_dir, 'xpidl')))
         self.assertTrue(os.path.isfile(os.path.join(install_dir, 'dist_idl')))
-        self.assertTrue(os.path.isfile(os.path.join(install_dir, 'xpidl')))
+
+        m = PurgeManifest(path=os.path.join(purge_dir, 'xpidl'))
+        self.assertIn('.deps/my_module.pp', m.entries)
+        self.assertIn('xpt/my_module.xpt', m.entries)
 
         m = InstallManifest(path=os.path.join(install_dir, 'dist_idl'))
         self.assertEqual(len(m), 2)
         self.assertIn('bar.idl', m)
         self.assertIn('foo.idl', m)
 
-        m = InstallManifest(path=os.path.join(install_dir, 'xpidl'))
-        self.assertIn('.deps/my_module.pp', m)
-        self.assertIn('xpt/my_module.xpt', m)
-
         m = InstallManifest(path=os.path.join(install_dir, 'dist_include'))
         self.assertIn('foo.h', m)
 
         p = os.path.join(env.topobjdir, 'config/makefiles/xpidl')
         self.assertTrue(os.path.isdir(p))
 
         self.assertTrue(os.path.isfile(os.path.join(p, 'Makefile')))
 
@@ -300,24 +302,45 @@ class TestRecursiveMakeBackend(BackendTe
         manifest_path = os.path.join(env.topobjdir,
             'testing', 'xpcshell', 'xpcshell.ini')
         lines = [l.strip() for l in open(manifest_path, 'rt').readlines()]
         expected = ('aa', 'bb', 'cc', 'dd', 'valid_val')
         self.assertEqual(lines, [
             '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.',
             ''] + ['[include:%s/xpcshell.ini]' % x for x in expected])
 
-    def test_old_install_manifest_deleted(self):
-        # Simulate an install manifest from a previous backend version. Ensure
-        # it is deleted.
+    def test_purge_manifests_written(self):
+        env = self._consume('stub0', RecursiveMakeBackend)
+
+        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
+        self.assertTrue(os.path.exists(purge_dir))
+
+        expected = [
+            'dist_bin',
+            'dist_private',
+            'dist_public',
+            'dist_sdk',
+            'tests',
+        ]
+
+        for e in expected:
+            full = os.path.join(purge_dir, e)
+            self.assertTrue(os.path.exists(full))
+
+        m = PurgeManifest(path=os.path.join(purge_dir, 'dist_bin'))
+        self.assertEqual(m.relpath, 'dist/bin')
+
+    def test_old_purge_manifest_deleted(self):
+        # Simulate a purge manifest from a previous backend version. Ensure it
+        # is deleted.
         env = self._get_environment('stub0')
-        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'install')
+        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
         manifest_path = os.path.join(purge_dir, 'old_manifest')
         os.makedirs(purge_dir)
-        m = InstallManifest()
+        m = PurgeManifest()
         m.write(path=manifest_path)
 
         self.assertTrue(os.path.exists(manifest_path))
         self._consume('stub0', RecursiveMakeBackend, env)
         self.assertFalse(os.path.exists(manifest_path))
 
     def test_install_manifests_written(self):
         env, objs = self._emit('stub0')
--- a/python/mozbuild/mozpack/manifests.py
+++ b/python/mozbuild/mozpack/manifests.py
@@ -29,16 +29,90 @@ def _auto_fileobj(path, fileobj, mode='r
 
     try:
         yield fileobj
     finally:
         if path:
             fileobj.close()
 
 
+class UnreadablePurgeManifest(Exception):
+    """Error for failure when reading content of a serialized PurgeManifest."""
+
+
+class PurgeManifest(object):
+    """Describes actions to be used with a copier.FilePurger instance.
+
+    This class facilitates serialization and deserialization of data used
+    to construct a copier.FilePurger and to perform a purge operation.
+
+    The manifest contains a set of entries (paths that are accounted for and
+    shouldn't be purged) and a relative path. The relative path is optional and
+    can be used to e.g. have several manifest files in a directory be
+    dynamically applied to subdirectories under a common base directory.
+
+    Don't be confused by the name of this class: entries are files that are
+    *not* purged.
+    """
+    def __init__(self, relpath='', path=None, fileobj=None):
+        self.relpath = relpath
+        self.entries = set()
+
+        if not path and not fileobj:
+            return
+
+        with _auto_fileobj(path, fileobj, mode='rt') as fh:
+            self._read_from_fh(fh)
+
+    def __eq__(self, other):
+        if not isinstance(other, PurgeManifest):
+            return False
+
+        return other.relpath == self.relpath and other.entries == self.entries
+
+    def _read_from_fh(self, fileobj):
+        version = fileobj.readline().rstrip()
+        if version != '1':
+            raise UnreadablePurgeManifest('Unknown manifest version: %s' %
+                version)
+
+        self.relpath = fileobj.readline().rstrip()
+
+        for entry in fileobj:
+            self.entries.add(entry.rstrip())
+
+    def add(self, path):
+        return self.entries.add(path)
+
+    def write(self, path=None, fileobj=None):
+        with _auto_fileobj(path, fileobj, 'wt') as fh:
+            fh.write('1\n')
+            fh.write('%s\n' % self.relpath)
+
+            # We write sorted so written output is consistent.
+            for entry in sorted(self.entries):
+                fh.write('%s\n' % entry)
+
+    def get_purger(self, prepend_relpath=False):
+        """Obtain a FilePurger instance from this manifest.
+
+        If :prepend_relpath is truish, the relative path in the manifest will
+        be prepended to paths added to the FilePurger. Otherwise, the raw paths
+        will be used.
+        """
+        p = FilePurger()
+        for entry in self.entries:
+            if prepend_relpath:
+                entry = mozpath.join(self.relpath, entry)
+
+            p.add(entry)
+
+        return p
+
+
 class UnreadableInstallManifest(Exception):
     """Raised when an invalid install manifest is parsed."""
 
 
 class InstallManifest(object):
     """Describes actions to be used with a copier.FileCopier instance.
 
     This class facilitates serialization and deserialization of data used to
--- a/python/mozbuild/mozpack/test/test_manifests.py
+++ b/python/mozbuild/mozpack/test/test_manifests.py
@@ -9,20 +9,53 @@ import os
 import mozunit
 
 from mozpack.copier import (
     FileCopier,
     FileRegistry,
 )
 from mozpack.manifests import (
     InstallManifest,
+    PurgeManifest,
+    UnreadablePurgeManifest,
 )
 from mozpack.test.test_files import TestWithTmpDir
 
 
+class TestPurgeManifest(TestWithTmpDir):
+    def test_construct(self):
+        m = PurgeManifest()
+        self.assertEqual(m.relpath, '')
+        self.assertEqual(len(m.entries), 0)
+
+    def test_serialization(self):
+        m = PurgeManifest(relpath='rel')
+        m.add('foo')
+        m.add('bar')
+        p = self.tmppath('m')
+        m.write(path=p)
+
+        self.assertTrue(os.path.exists(p))
+
+        m2 = PurgeManifest(path=p)
+        self.assertEqual(m.relpath, m2.relpath)
+        self.assertEqual(m.entries, m2.entries)
+        self.assertEqual(m, m2)
+
+    def test_unknown_version(self):
+        p = self.tmppath('bad')
+
+        with open(p, 'wt') as fh:
+            fh.write('2\n')
+            fh.write('not relevant')
+
+        with self.assertRaises(UnreadablePurgeManifest):
+            PurgeManifest(path=p)
+
+
 class TestInstallManifest(TestWithTmpDir):
     def test_construct(self):
         m = InstallManifest()
         self.assertEqual(len(m), 0)
 
     def test_adds(self):
         m = InstallManifest()
         m.add_symlink('s_source', 's_dest')