Backed out 2 changesets (bug 884587) because of Windows build bustage on a CLOSED TREE
authorEhsan Akhgari <ehsan@mozilla.com>
Tue, 25 Jun 2013 16:38:55 -0400
changeset 136411 301c184ae8e7ade65777e41617c43a4c3b5030ac
parent 136410 3ccbd6d8354e1ed9e987cd1113abd83fc99613c2
child 136412 1d07c8c757742802d80b8534b2dd804b1d84811f
push id30098
push usereakhgari@mozilla.com
push dateTue, 25 Jun 2013 20:39:09 +0000
treeherdermozilla-inbound@301c184ae8e7 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs884587
milestone25.0a1
backs out0244a34bc4190998441c143140451598fe941c66
796961a384b47798de261e2c9c6b331f39144f1c
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out 2 changesets (bug 884587) because of Windows build bustage on a CLOSED TREE Backed out changeset 0244a34bc419 (bug 884587) Backed out changeset 796961a384b4 (bug 884587)
Makefile.in
config/purge_directories.py
python/mozbuild/mozbuild/backend/recursivemake.py
python/mozbuild/mozbuild/test/backend/common.py
python/mozbuild/mozbuild/test/backend/test_recursivemake.py
python/mozbuild/mozpack/manifests.py
python/mozbuild/mozpack/test/test_manifests.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -33,39 +33,25 @@ include $(topsrcdir)/config/config.mk
 GARBAGE_DIRS += dist _javagen _profile _tests staticlib
 DIST_GARBAGE = config.cache config.log config.status* config-defs.h \
    config/autoconf.mk \
    unallmakefiles mozilla-config.h \
    netwerk/necko-config.h xpcom/xpcom-config.h xpcom/xpcom-private.h \
    $(topsrcdir)/.mozconfig.mk $(topsrcdir)/.mozconfig.out
 
 ifndef MOZ_PROFILE_USE
-# One of the first things we do in the build is purge "unknown" files
-# from the object directory. This serves two purposes:
-#
-#   1) Remove files from a previous build no longer accounted for in
-#      this build configuration.
-#
-#   2) Work around poor build system dependencies by forcing some
-#      rebuilds.
-#
-# Ideally #2 does not exist. Our reliance on this aspect should diminish
-# over time.
-#
-# moz.build backend generation simply installs a set of "manifests" into
-# a common directory. Each manifest is responsible for defining files in
-# a specific subdirectory of the object directory. The invoked Python
-# script simply iterates over all the manifests, purging files as
-# necessary. To manage new directories or add files to the manifests,
-# modify the backend generator.
-#
 # We need to explicitly put backend.RecursiveMakeBackend.built here
 # otherwise the rule in rules.mk doesn't run early enough.
 default alldep all:: CLOBBER $(topsrcdir)/configure config.status backend.RecursiveMakeBackend.built
-	$(PYTHON) $(topsrcdir)/config/purge_directories.py -d _build_manifests/purge .
+	$(RM) -r $(DIST)/sdk
+	$(RM) -r $(DIST)/include
+	$(RM) -r $(DIST)/private
+	$(RM) -r $(DIST)/public
+	$(RM) -r $(DIST)/bin
+	$(RM) -r _tests
 endif
 
 CLOBBER: $(topsrcdir)/CLOBBER
 	@echo "STOP!  The CLOBBER file has changed."
 	@echo "Please run the build through a sanctioned build wrapper, such as"
 	@echo "'mach build' or client.mk."
 	@exit 1
 
deleted file mode 100644
--- a/config/purge_directories.py
+++ /dev/null
@@ -1,77 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-# This script is used to purge a directory of unwanted files as defined by
-# a manifest file.
-
-from __future__ import print_function, unicode_literals
-
-import argparse
-import os
-import sys
-import threading
-
-from mozpack.manifests import PurgeManifest
-
-def do_purge(purger, dest, state):
-    state['result'] = purger.purge(dest)
-
-def process_manifest(topdir, manifest_path):
-    manifest = PurgeManifest.from_path(manifest_path)
-    purger = manifest.get_purger()
-    full = os.path.join(topdir, manifest.relpath)
-
-    state = dict(
-        relpath=manifest.relpath,
-        result=None,
-    )
-
-    t = threading.Thread(target=do_purge, args=(purger, full, state))
-    state['thread'] = t
-    t.start()
-
-    return state
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(
-        description='Purge a directory of untracked files.')
-
-    parser.add_argument('--directory', '-d',
-        help='Directory containing manifest files. Will process every file '
-            'in directory.')
-    parser.add_argument('topdir',
-        help='Top directory all paths are evaluated from.')
-    parser.add_argument('manifests', nargs='*',
-        help='List of manifest files defining purge operations to perform.')
-
-    args = parser.parse_args()
-
-    states = []
-
-    print('Purging unaccounted files from object directory...')
-
-    # We perform purging using threads for performance reasons. Hopefully
-    # multiple I/O operations will be faster than just 1.
-    paths = []
-    if args.directory:
-        for path in sorted(os.listdir(args.directory)):
-            paths.append(os.path.join(args.directory, path))
-
-    paths.extend(args.manifests)
-
-    for path in paths:
-        states.append(process_manifest(args.topdir, path))
-
-    for state in states:
-        state['thread'].join()
-        print('Deleted %d files and %d directories from %s.' % (
-            state['result'].removed_files_count,
-            state['result'].removed_directories_count,
-            state['relpath']
-        ))
-
-    print('Finished purging.')
-
-    sys.exit(0)
--- a/python/mozbuild/mozbuild/backend/recursivemake.py
+++ b/python/mozbuild/mozbuild/backend/recursivemake.py
@@ -4,19 +4,16 @@
 
 from __future__ import unicode_literals
 
 import errno
 import logging
 import os
 import types
 
-from mozpack.copier import FilePurger
-from mozpack.manifests import PurgeManifest
-
 from .base import BuildBackend
 from ..frontend.data import (
     ConfigFileSubstitution,
     DirectoryTraversal,
     SandboxDerived,
     VariablePassthru,
     Exports,
     Program,
@@ -125,25 +122,16 @@ class RecursiveMakeBackend(BuildBackend)
         self.summary.backend_detailed_summary = types.MethodType(detailed,
             self.summary)
 
         self.xpcshell_manifests = []
 
         self.backend_input_files.add(os.path.join(self.environment.topobjdir,
             'config', 'autoconf.mk'))
 
-        self._purge_manifests = dict(
-            dist_bin=PurgeManifest(relpath='dist/bin'),
-            dist_include=PurgeManifest(relpath='dist/include'),
-            dist_private=PurgeManifest(relpath='dist/private'),
-            dist_public=PurgeManifest(relpath='dist/public'),
-            dist_sdk=PurgeManifest(relpath='dist/sdk'),
-            tests=PurgeManifest(relpath='_tests'),
-        )
-
     def _update_from_avoid_write(self, result):
         existed, updated = result
 
         if not existed:
             self.summary.created_count += 1
         elif updated:
             self.summary.updated_count += 1
         else:
@@ -259,18 +247,16 @@ class RecursiveMakeBackend(BuildBackend)
                 self.environment.topobjdir, 'testing', 'xpcshell', 'xpcshell.ini'))
             mastermanifest.write(
                 '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.\n\n')
             for manifest in self.xpcshell_manifests:
                 mastermanifest.write("[include:%s]\n" % manifest)
             self._update_from_avoid_write(mastermanifest.close())
             self.summary.managed_count += 1
 
-        self._write_purge_manifests()
-
     def _process_directory_traversal(self, obj, backend_file):
         """Process a data.DirectoryTraversal instance."""
         fh = backend_file.fh
 
         for tier, dirs in obj.tier_dirs.iteritems():
             fh.write('TIERS += %s\n' % tier)
 
             if dirs:
@@ -318,50 +304,22 @@ class RecursiveMakeBackend(BuildBackend)
         else:
             export_name = 'EXPORTS'
 
         # Iterate over the list of export filenames, printing out an EXPORTS
         # declaration for each.
         if strings:
             backend_file.write('%s += %s\n' % (export_name, ' '.join(strings)))
 
-            for s in strings:
-                p = '%s%s' % (namespace, s)
-                self._purge_manifests['dist_include'].add(p)
-
         children = exports.get_children()
         for subdir in sorted(children):
             self._process_exports(children[subdir], backend_file,
                                   namespace=namespace + subdir)
 
     def _process_program(self, program, backend_file):
         backend_file.write('PROGRAM = %s\n' % program)
 
     def _process_xpcshell_manifests(self, obj, backend_file, namespace=""):
         manifest = obj.xpcshell_manifests
         backend_file.write('XPCSHELL_TESTS += %s\n' % os.path.dirname(manifest))
         if obj.relativedir != '':
             manifest = '%s/%s' % (obj.relativedir, manifest)
         self.xpcshell_manifests.append(manifest)
-
-    def _write_purge_manifests(self):
-        # We write out a "manifest" file for each directory that is to be
-        # purged.
-        #
-        # Ideally we have as few manifests as possible - ideally only 1. This
-        # will likely require all build metadata to be in emitted objects.
-        # We're not quite there yet, so we maintain multiple manifests.
-        man_dir = os.path.join(self.environment.topobjdir, '_build_manifests',
-            'purge')
-
-        # We have a purger for the manifests themselves to ensure we don't over
-        # purge if we delete a purge manifest.
-        purger = FilePurger()
-
-        for k, manifest in self._purge_manifests.items():
-            purger.add(k)
-            full = os.path.join(man_dir, k)
-
-            fh = FileAvoidWrite(os.path.join(man_dir, k))
-            manifest.write_fileobj(fh)
-            self._update_from_avoid_write(fh.close())
-
-        purger.purge(man_dir)
--- a/python/mozbuild/mozbuild/test/backend/common.py
+++ b/python/mozbuild/mozbuild/test/backend/common.py
@@ -84,27 +84,25 @@ class BackendTester(unittest.TestCase):
 
         objdir = mkdtemp()
         self.addCleanup(rmtree, objdir)
 
         srcdir = os.path.join(test_data_path, name)
         config['substs'].append(('top_srcdir', srcdir))
         return ConfigEnvironment(srcdir, objdir, **config)
 
-    def _emit(self, name, env=None):
-        if not env:
-            env = self._get_environment(name)
-
+    def _emit(self, name):
+        env = self._get_environment(name)
         reader = BuildReader(env)
         emitter = TreeMetadataEmitter(env)
 
         return env, emitter.emit(reader.read_topsrcdir())
 
-    def _consume(self, name, cls, env=None):
-        env, objs = self._emit(name, env=env)
+    def _consume(self, name, cls):
+        env, objs = self._emit(name)
         backend = cls(env)
         backend.consume(objs)
 
         return env
 
     def _tree_paths(self, topdir, filename):
         for dirpath, dirnames, filenames in os.walk(topdir):
             for f in filenames:
--- a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
+++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py
@@ -2,17 +2,16 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 import os
 import time
 
-from mozpack.manifests import PurgeManifest
 from mozunit import main
 
 from mozbuild.backend.configenvironment import ConfigEnvironment
 from mozbuild.backend.recursivemake import RecursiveMakeBackend
 from mozbuild.frontend.emitter import TreeMetadataEmitter
 from mozbuild.frontend.reader import BuildReader
 
 from mozbuild.test.backend.common import BackendTester
@@ -226,23 +225,16 @@ class TestRecursiveMakeBackend(BackendTe
             'EXPORTS_NAMESPACES += mozilla/dom',
             'EXPORTS_mozilla/dom += dom1.h dom2.h',
             'EXPORTS_NAMESPACES += mozilla/gfx',
             'EXPORTS_mozilla/gfx += gfx.h',
             'EXPORTS_NAMESPACES += nspr/private',
             'EXPORTS_nspr/private += pprio.h',
         ])
 
-        # EXPORTS files should appear in the dist_include purge manifest.
-        m = PurgeManifest.from_path(os.path.join(env.topobjdir,
-            '_build_manifests', 'purge', 'dist_include'))
-        self.assertIn('foo.h', m.entries)
-        self.assertIn('mozilla/mozilla1.h', m.entries)
-        self.assertIn('mozilla/dom/dom2.h', m.entries)
-
     def test_xpcshell_manifests(self):
         """Ensure XPCSHELL_TESTS_MANIFESTS is written out correctly."""
         env = self._consume('xpcshell_manifests', RecursiveMakeBackend)
 
         backend_path = os.path.join(env.topobjdir, 'backend.mk')
         lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]]
 
         # Avoid positional parameter and async related breakage
@@ -260,47 +252,10 @@ class TestRecursiveMakeBackend(BackendTe
         manifest_path = os.path.join(env.topobjdir,
             'testing', 'xpcshell', 'xpcshell.ini')
         lines = [l.strip() for l in open(manifest_path, 'rt').readlines()]
         expected = ('aa', 'bb', 'cc', 'dd', 'valid_val')
         self.assertEqual(lines, [
             '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.',
             ''] + ['[include:%s/xpcshell.ini]' % x for x in expected])
 
-    def test_purge_manifests_written(self):
-        env = self._consume('stub0', RecursiveMakeBackend)
-
-        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
-        self.assertTrue(os.path.exists(purge_dir))
-
-        expected = [
-            'dist_bin',
-            'dist_include',
-            'dist_private',
-            'dist_public',
-            'dist_sdk',
-            'tests',
-        ]
-
-        for e in expected:
-            full = os.path.join(purge_dir, e)
-            self.assertTrue(os.path.exists(full))
-
-        m = PurgeManifest.from_path(os.path.join(purge_dir, 'dist_bin'))
-        self.assertEqual(m.relpath, 'dist/bin')
-
-    def test_old_purge_manifest_deleted(self):
-        # Simulate a purge manifest from a previous backend version. Ensure it
-        # is deleted.
-        env = self._get_environment('stub0')
-        purge_dir = os.path.join(env.topobjdir, '_build_manifests', 'purge')
-        manifest_path = os.path.join(purge_dir, 'old_manifest')
-        os.makedirs(purge_dir)
-        m = PurgeManifest()
-        m.write_file(manifest_path)
-
-        self.assertTrue(os.path.exists(manifest_path))
-        self._consume('stub0', RecursiveMakeBackend, env)
-        self.assertFalse(os.path.exists(manifest_path))
-
-
 if __name__ == '__main__':
     main()
deleted file mode 100644
--- a/python/mozbuild/mozpack/manifests.py
+++ /dev/null
@@ -1,89 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import unicode_literals
-
-from .copier import FilePurger
-import mozpack.path as mozpath
-
-
-class UnreadablePurgeManifest(Exception):
-    """Error for failure when reading content of a serialized PurgeManifest."""
-
-
-class PurgeManifest(object):
-    """Describes actions to be used with a copier.FilePurger instance.
-
-    This class facilitates serialization and deserialization of data used
-    to construct a copier.FilePurger and to perform a purge operation.
-
-    The manifest contains a set of entries (paths that are accounted for and
-    shouldn't be purged) and a relative path. The relative path is optional and
-    can be used to e.g. have several manifest files in a directory be
-    dynamically applied to subdirectories under a common base directory.
-
-    Don't be confused by the name of this class: entries are files that are
-    *not* purged.
-    """
-    def __init__(self, relpath=''):
-        self.relpath = relpath
-        self.entries = set()
-
-    def __eq__(self, other):
-        if not isinstance(other, PurgeManifest):
-            return False
-
-        return other.relpath == self.relpath and other.entries == self.entries
-
-    @staticmethod
-    def from_path(path):
-        with open(path, 'rt') as fh:
-            return PurgeManifest.from_fileobj(fh)
-
-    @staticmethod
-    def from_fileobj(fh):
-        m = PurgeManifest()
-
-        version = fh.readline().rstrip()
-        if version != '1':
-            raise UnreadablePurgeManifest('Unknown manifest version: ' %
-                version)
-
-        m.relpath = fh.readline().rstrip()
-
-        for entry in fh:
-            m.entries.add(entry.rstrip())
-
-        return m
-
-    def add(self, path):
-        return self.entries.add(path)
-
-    def write_file(self, path):
-        with open(path, 'wt') as fh:
-            return self.write_fileobj(fh)
-
-    def write_fileobj(self, fh):
-        fh.write('1\n')
-        fh.write('%s\n' % self.relpath)
-
-        # We write sorted so written output is consistent.
-        for entry in sorted(self.entries):
-            fh.write('%s\n' % entry)
-
-    def get_purger(self, prepend_relpath=False):
-        """Obtain a FilePurger instance from this manifest.
-
-        If :prepend_relpath is truish, the relative path in the manifest will
-        be prepended to paths added to the FilePurger. Otherwise, the raw paths
-        will be used.
-        """
-        p = FilePurger()
-        for entry in self.entries:
-            if prepend_relpath:
-                entry = mozpath.join(self.relpath, entry)
-
-            p.add(entry)
-
-        return p
deleted file mode 100644
--- a/python/mozbuild/mozpack/test/test_manifests.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import unicode_literals
-
-import os
-import unittest
-
-import mozunit
-
-from mozpack.manifests import (
-    PurgeManifest,
-    UnreadablePurgeManifest,
-)
-from mozpack.test.test_files import TestWithTmpDir
-
-
-class TestPurgeManifest(TestWithTmpDir):
-    def test_construct(self):
-        m = PurgeManifest()
-        self.assertEqual(m.relpath, '')
-        self.assertEqual(len(m.entries), 0)
-
-    def test_serialization(self):
-        m = PurgeManifest(relpath='rel')
-        m.add('foo')
-        m.add('bar')
-        p = self.tmppath('m')
-        m.write_file(p)
-
-        self.assertTrue(os.path.exists(p))
-
-        m2 = PurgeManifest.from_path(p)
-        self.assertEqual(m.relpath, m2.relpath)
-        self.assertEqual(m.entries, m2.entries)
-        self.assertEqual(m, m2)
-
-    def test_unknown_version(self):
-        p = self.tmppath('bad')
-
-        with open(p, 'wt') as fh:
-            fh.write('2\n')
-            fh.write('not relevant')
-
-        with self.assertRaises(UnreadablePurgeManifest):
-            PurgeManifest.from_path(p)
-