Backed out changesets 206bb159d62f, a6ab3e11f721, and a61aa08ab4ac (bug 837323) for frequent Windows fatal clobber errors on a CLOSED TREE.
authorRyan VanderMeulen <ryanvm@gmail.com>
Mon, 08 Apr 2013 09:59:45 -0400
changeset 138931 27211859cb34dd4bd369d45c5274d734a09b255e
parent 138930 b8078621fb2a6e8502fd8fd134737482453d30e6
child 138932 e19ace7a61e4817da259b5a2a9ca928b5033ae8a
push id2579
push userakeybl@mozilla.com
push dateMon, 24 Jun 2013 18:52:47 +0000
treeherdermozilla-beta@b69b7de8a05a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs837323
milestone23.0a1
backs out206bb159d62ffb02f39640cb7c4ec68903f27356
a6ab3e11f721e49e0b2499ba0fa54ecf80e35522
a61aa08ab4ac92a5f909a87f0aa717a05eb36df2
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out changesets 206bb159d62f, a6ab3e11f721, and a61aa08ab4ac (bug 837323) for frequent Windows fatal clobber errors on a CLOSED TREE.
Makefile.in
client.mk
configure.in
python/Makefile.in
python/mozbuild/mozbuild/controller/__init__.py
python/mozbuild/mozbuild/controller/clobber.py
python/mozbuild/mozbuild/test/controller/__init__.py
python/mozbuild/mozbuild/test/controller/test_clobber.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -30,30 +30,24 @@ include $(topsrcdir)/config/config.mk
 
 GARBAGE_DIRS += dist _javagen _profile _tests staticlib
 DIST_GARBAGE = config.cache config.log config.status* config-defs.h \
    config/autoconf.mk \
    unallmakefiles mozilla-config.h \
    netwerk/necko-config.h xpcom/xpcom-config.h xpcom/xpcom-private.h \
    $(topsrcdir)/.mozconfig.mk $(topsrcdir)/.mozconfig.out
 
-default alldep all:: CLOBBER $(topsrcdir)/configure config.status
+default alldep all:: $(topsrcdir)/configure config.status
 	$(RM) -r $(DIST)/sdk
 	$(RM) -r $(DIST)/include
 	$(RM) -r $(DIST)/private
 	$(RM) -r $(DIST)/public
 	$(RM) $(DIST)/bin/chrome.manifest $(DIST)/bin/components/components.manifest
 	$(RM) -r _tests
 
-CLOBBER: $(topsrcdir)/CLOBBER
-	@echo "STOP!  The CLOBBER file has changed."
-	@echo "Please run the build through a sanctioned build wrapper, such as"
-	@echo "'mach build' or client.mk."
-	@exit 1
-
 $(topsrcdir)/configure: $(topsrcdir)/configure.in
 	@echo "STOP!  configure.in has changed, and your configure is out of date."
 	@echo "Please rerun autoconf and re-configure your build directory."
 	@echo "To ignore this message, touch 'configure' in the source directory,"
 	@echo "but your build might not succeed."
 	@exit 1
 
 config.status: $(topsrcdir)/configure
--- a/client.mk
+++ b/client.mk
@@ -103,19 +103,16 @@ define CR
 
 endef
 
 # As $(shell) doesn't preserve newlines, use sed to replace them with an
 # unlikely sequence (||), which is then replaced back to newlines by make
 # before evaluation.
 $(eval $(subst ||,$(CR),$(shell _PYMAKE=$(.PYMAKE) $(TOPSRCDIR)/$(MOZCONFIG_LOADER) $(TOPSRCDIR) 2> $(TOPSRCDIR)/.mozconfig.out | sed 's/$$/||/')))
 
-ifdef NO_AUTOCLOBBER
-export NO_AUTOCLOBBER=1
-endif
 
 # Automatically add -jN to make flags if not defined. N defaults to number of cores.
 ifeq (,$(findstring -j,$(MOZ_MAKE_FLAGS)))
   cores=$(shell $(PYTHON) -c 'import multiprocessing; print(multiprocessing.cpu_count())')
   MOZ_MAKE_FLAGS += -j$(cores)
 endif
 
 
@@ -300,23 +297,19 @@ CONFIGURE_ENV_ARGS += \
 #   $(TOPSRCDIR) will set @srcdir@ to "."; otherwise, it is set to the full
 #   path of $(TOPSRCDIR).
 ifeq ($(TOPSRCDIR),$(OBJDIR))
   CONFIGURE = ./configure
 else
   CONFIGURE = $(TOPSRCDIR)/configure
 endif
 
-check-clobber:
-	$(PYTHON) $(TOPSRCDIR)/python/mozbuild/mozbuild/controller/clobber.py $(TOPSRCDIR) $(OBJDIR)
-
 configure-files: $(CONFIGURES)
 
 configure-preqs = \
-  check-clobber \
   configure-files \
   $(call mkdir_deps,$(OBJDIR)) \
   $(if $(MOZ_BUILD_PROJECTS),$(call mkdir_deps,$(MOZ_OBJDIR))) \
   save-mozconfig \
   $(NULL)
 
 save-mozconfig: $(FOUND_MOZCONFIG)
 	-cp $(FOUND_MOZCONFIG) $(OBJDIR)/.mozconfig
@@ -444,28 +437,9 @@ check-sync-dirs-%:
 echo-variable-%:
 	@echo $($*)
 
 # This makefile doesn't support parallel execution. It does pass
 # MOZ_MAKE_FLAGS to sub-make processes, so they will correctly execute
 # in parallel.
 .NOTPARALLEL:
 
-.PHONY: checkout \
-    real_checkout \
-    depend \
-    realbuild \
-    build \
-    profiledbuild \
-    cleansrcdir \
-    pull_all \
-    build_all \
-    check-clobber \
-    clobber \
-    clobber_all \
-    pull_and_build_all \
-    everything \
-    configure \
-    preflight_all \
-    preflight \
-    postflight \
-    postflight_all \
-    $(OBJDIR_TARGETS)
+.PHONY: checkout real_checkout depend realbuild build profiledbuild cleansrcdir pull_all build_all clobber clobber_all pull_and_build_all everything configure preflight_all preflight postflight postflight_all $(OBJDIR_TARGETS)
--- a/configure.in
+++ b/configure.in
@@ -115,16 +115,40 @@ then
 	***
 	EOF
     exit 1
     break
   fi
 fi
 MOZ_BUILD_ROOT=`pwd`
 
+dnl Do not allow building if a clobber is required
+dnl ==============================================================
+dnl TODO Make this better, ideally this would clobber automaticially
+if test -e $_objdir/CLOBBER; then
+  if test $_topsrcdir/CLOBBER -nt $_objdir/CLOBBER; then
+    echo "	***"
+    echo "	*	The CLOBBER file has been updated, indicating that an incremental build"
+    echo "	*	since your last build will probably not work. A full build is required."
+    echo "	*	The change that caused this is:"
+    cat $_topsrcdir/CLOBBER | sed '/^#/d' | sed 's/^/	*	/'
+    echo "	*	"
+    echo "	*	The easiest way to fix this is to manually delete your objdir:"
+    echo "	*	rm -rf $_objdir"
+    echo "	*	"
+    echo "	*	Or, if you know this clobber doesn't apply to you, it can be ignored with:"
+    echo "	*	cp '$_topsrcdir/CLOBBER' $_objdir"
+    echo "	***"
+    exit 1
+    break;
+  fi
+else
+  touch $_objdir/CLOBBER
+fi
+
 MOZ_PYTHON
 
 MOZ_DEFAULT_COMPILER
 
 COMPILE_ENVIRONMENT=1
 MOZ_ARG_DISABLE_BOOL(compile-environment,
 [  --disable-compile-environment
                           Disable compiler/library checks.],
--- a/python/Makefile.in
+++ b/python/Makefile.in
@@ -7,17 +7,16 @@ topsrcdir := @top_srcdir@
 srcdir := @srcdir@
 VPATH = @srcdir@
 
 include $(DEPTH)/config/autoconf.mk
 
 test_dirs := \
   mozbuild/mozbuild/test \
   mozbuild/mozbuild/test/backend \
-  mozbuild/mozbuild/test/controller \
   mozbuild/mozbuild/test/compilation \
   mozbuild/mozbuild/test/frontend \
   mozbuild/mozpack/test \
   $(NULL)
 
 PYTHON_UNIT_TESTS := $(foreach dir,$(test_dirs),$(wildcard $(srcdir)/$(dir)/*.py))
 
 include $(topsrcdir)/config/rules.mk
deleted file mode 100644
deleted file mode 100644
--- a/python/mozbuild/mozbuild/controller/clobber.py
+++ /dev/null
@@ -1,194 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import print_function
-
-r'''This module contains code for managing clobbering of the tree.'''
-
-import os
-import sys
-
-try:
-    from mozfile.mozfile import rmtree
-except ImportError:
-    from shutil import rmtree
-
-
-CLOBBER_MESSAGE = '''
-***
-* The CLOBBER file has been updated, indicating that an incremental build since
-* your last build will probably not work. A full/clobber build is required.
-*
-* The reason for the clobber is:
-*
-{clobber_reason}
-*
-* Clobbering can be performed automatically. However, we didn't automatically
-* clobber this time because:
-*
-*   {no_reason}
-*
-* The easiest and fastest way to clobber is to run:
-*
-*  $ mach clobber
-*
-* If you know this clobber doesn't apply to you or you're feeling lucky - well
-* do ya? - you can ignore this clobber requirement by running:
-*
-*  $ touch {clobber_file}
-*
-***
-'''.strip()
-
-
-class Clobberer(object):
-    def __init__(self, topsrcdir, topobjdir):
-        """Create a new object to manage clobbering the tree.
-
-        It is bound to a top source directory and to a specific object
-        directory.
-        """
-        assert os.path.isabs(topsrcdir)
-        assert os.path.isabs(topobjdir)
-
-        self.topsrcdir = os.path.normpath(topsrcdir)
-        self.topobjdir = os.path.normpath(topobjdir)
-        self.src_clobber = os.path.join(topsrcdir, 'CLOBBER')
-        self.obj_clobber = os.path.join(topobjdir, 'CLOBBER')
-
-        assert os.path.isfile(self.src_clobber)
-
-    def clobber_needed(self):
-        """Returns a bool indicating whether a tree clobber is required."""
-
-        # No object directory clobber file means we're good.
-        if not os.path.exists(self.obj_clobber):
-            return False
-
-        # Object directory clobber older than current is fine.
-        if os.path.getmtime(self.src_clobber) <= \
-            os.path.getmtime(self.obj_clobber):
-
-            return False
-
-        return True
-
-    def clobber_cause(self):
-        """Obtain the cause why a clobber is required.
-
-        This reads the cause from the CLOBBER file.
-
-        This returns a list of lines describing why the clobber was required.
-        Each line is stripped of leading and trailing whitespace.
-        """
-        with open(self.src_clobber, 'rt') as fh:
-            lines = [l.strip() for l in fh.readlines()]
-            return [l for l in lines if l and not l.startswith('#')]
-
-    def ensure_objdir_state(self):
-        """Ensure the CLOBBER file in the objdir exists.
-
-        This is called as part of the build to ensure the clobber information
-        is configured properly for the objdir.
-        """
-        if not os.path.exists(self.topobjdir):
-            os.makedirs(self.topobjdir)
-
-        if not os.path.exists(self.obj_clobber):
-            # Simply touch the file.
-            with open(self.obj_clobber, 'a'):
-                pass
-
-    def maybe_do_clobber(self, cwd, allow_auto=True, fh=sys.stderr):
-        """Perform a clobber if it is required. Maybe.
-
-        This is the API the build system invokes to determine if a clobber
-        is needed and to automatically perform that clobber if we can.
-
-        This returns a tuple of (bool, bool, str). The elements are:
-
-          - Whether a clobber was/is required.
-          - Whether a clobber was performed.
-          - The reason why the clobber failed or could not be performed. This
-            will be None if no clobber is required or if we clobbered without
-            error.
-        """
-        assert cwd
-        cwd = os.path.normpath(cwd)
-
-        if not self.clobber_needed():
-            print('Clobber not needed.', file=fh)
-            self.ensure_objdir_state()
-            return False, False, None
-
-        # So a clobber is needed. We only perform a clobber if we are
-        # allowed to perform an automatic clobber (the default) and if the
-        # current directory is not under the object directory. The latter is
-        # because operating systems, filesystems, and shell can throw fits
-        # if the current working directory is deleted from under you. While it
-        # can work in some scenarios, we take the conservative approach and
-        # never try.
-        if not allow_auto:
-            return True, False, self._message(
-                'Automatic clobbering has been disabled.')
-
-        if cwd.startswith(self.topobjdir) and cwd != self.topobjdir:
-            return True, False, self._message(
-                'Cannot clobber while the shell is inside the object directory.')
-
-        print('Automatically clobbering %s' % self.topobjdir, file=fh)
-        try:
-            if cwd == self.topobjdir:
-                for entry in os.listdir(self.topobjdir):
-                    full = os.path.join(self.topobjdir, entry)
-
-                    if os.path.isdir(full):
-                        rmtree(full)
-                    else:
-                        os.unlink(full)
-
-            else:
-                rmtree(self.topobjdir)
-
-            self.ensure_objdir_state()
-            print('Successfully completed auto clobber.', file=fh)
-            return True, True, None
-        except (IOError) as error:
-            return True, False, self._message(
-                'Error when automatically clobbering: ' + str(error))
-
-    def _message(self, reason):
-        lines = ['*  ' + line for line in self.clobber_cause()]
-
-        return CLOBBER_MESSAGE.format(clobber_reason='\n'.join(lines),
-            no_reason=reason, clobber_file=self.obj_clobber)
-
-
-def main(args, env, cwd, fh=sys.stderr):
-    if len(args) != 2:
-        print('Usage: clobber.py topsrcdir topobjdir', file=fh)
-        return 1
-
-    topsrcdir, topobjdir = args
-
-    if not os.path.isabs(topsrcdir):
-        topsrcdir = os.path.abspath(topsrcdir)
-
-    if not os.path.isabs(topobjdir):
-        topobjdir = os.path.abspath(topobjdir)
-
-    auto = False if env.get('NO_AUTOCLOBBER', False) else True
-    clobber = Clobberer(topsrcdir, topobjdir)
-    required, performed, message = clobber.maybe_do_clobber(cwd, auto, fh)
-
-    if not required or performed:
-        return 0
-
-    print(message, file=fh)
-    return 1
-
-
-if __name__ == '__main__':
-    sys.exit(main(sys.argv[1:], os.environ, os.getcwd(), sys.stdout))
-
deleted file mode 100644
deleted file mode 100644
--- a/python/mozbuild/mozbuild/test/controller/test_clobber.py
+++ /dev/null
@@ -1,208 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import unicode_literals
-
-import os
-import shutil
-import tempfile
-import unittest
-
-from StringIO import StringIO
-
-from mozunit import main
-
-from mozbuild.controller.clobber import Clobberer
-from mozbuild.controller.clobber import main as clobber
-
-
-class TestClobberer(unittest.TestCase):
-    def setUp(self):
-        self._temp_dirs = []
-
-        return unittest.TestCase.setUp(self)
-
-    def tearDown(self):
-        for d in self._temp_dirs:
-            shutil.rmtree(d, ignore_errors=True)
-
-        return unittest.TestCase.tearDown(self)
-
-    def get_tempdir(self):
-        t = tempfile.mkdtemp()
-        self._temp_dirs.append(t)
-        return t
-
-    def get_topsrcdir(self):
-        t = self.get_tempdir()
-        p = os.path.join(t, 'CLOBBER')
-        with open(p, 'a'):
-            pass
-
-        return t
-
-    def test_no_objdir(self):
-        """If topobjdir does not exist, no clobber is needed."""
-
-        tmp = os.path.join(self.get_tempdir(), 'topobjdir')
-        self.assertFalse(os.path.exists(tmp))
-
-        c = Clobberer(self.get_topsrcdir(), tmp)
-        self.assertFalse(c.clobber_needed())
-
-        # Side-effect is topobjdir is created with CLOBBER file touched.
-        required, performed, reason = c.maybe_do_clobber(os.getcwd())
-        self.assertFalse(required)
-        self.assertFalse(performed)
-        self.assertIsNone(reason)
-
-        self.assertTrue(os.path.isdir(tmp))
-        self.assertTrue(os.path.exists(os.path.join(tmp, 'CLOBBER')))
-
-    def test_objdir_no_clobber_file(self):
-        """If CLOBBER does not exist in topobjdir, treat as empty."""
-
-        c = Clobberer(self.get_topsrcdir(), self.get_tempdir())
-        self.assertFalse(c.clobber_needed())
-
-        required, performed, reason = c.maybe_do_clobber(os.getcwd())
-        self.assertFalse(required)
-        self.assertFalse(performed)
-        self.assertIsNone(reason)
-
-        self.assertTrue(os.path.exists(os.path.join(c.topobjdir, 'CLOBBER')))
-
-    def test_objdir_clobber_newer(self):
-        """If CLOBBER in topobjdir is newer, do nothing."""
-
-        c = Clobberer(self.get_topsrcdir(), self.get_tempdir())
-        with open(c.obj_clobber, 'a'):
-            pass
-
-        required, performed, reason = c.maybe_do_clobber(os.getcwd())
-        self.assertFalse(required)
-        self.assertFalse(performed)
-        self.assertIsNone(reason)
-
-    def test_objdir_clobber_older(self):
-        """If CLOBBER in topobjdir is older, we clobber."""
-
-        c = Clobberer(self.get_topsrcdir(), self.get_tempdir())
-        with open(c.obj_clobber, 'a'):
-            pass
-
-        dummy_path = os.path.join(c.topobjdir, 'foo')
-        with open(dummy_path, 'a'):
-            pass
-
-        self.assertTrue(os.path.exists(dummy_path))
-
-        old_time = os.path.getmtime(c.src_clobber) - 60
-        os.utime(c.obj_clobber, (old_time, old_time))
-
-        self.assertTrue(c.clobber_needed())
-
-        required, performed, reason = c.maybe_do_clobber(os.getcwd(), False)
-        self.assertTrue(required)
-        self.assertFalse(performed)
-        self.assertIn('Automatic clobbering has been disabled', reason)
-
-        # Now let's actually do it.
-        required, performed, reason = c.maybe_do_clobber(os.getcwd())
-        self.assertTrue(required)
-        self.assertTrue(performed)
-
-        self.assertFalse(os.path.exists(dummy_path))
-        self.assertTrue(os.path.exists(c.obj_clobber))
-        self.assertGreaterEqual(os.path.getmtime(c.obj_clobber),
-            os.path.getmtime(c.src_clobber))
-
-    def test_objdir_is_srcdir(self):
-        """If topobjdir is the topsrcdir, refuse to clobber."""
-
-        tmp = self.get_topsrcdir()
-        c = Clobberer(tmp, tmp)
-
-        self.assertFalse(c.clobber_needed())
-
-    def test_cwd_is_topobjdir(self):
-        """If cwd is topobjdir, we can still clobber."""
-        c = Clobberer(self.get_topsrcdir(), self.get_tempdir())
-
-        with open(c.obj_clobber, 'a'):
-            pass
-
-        dummy_file = os.path.join(c.topobjdir, 'dummy_file')
-        with open(dummy_file, 'a'):
-            pass
-
-        dummy_dir = os.path.join(c.topobjdir, 'dummy_dir')
-        os.mkdir(dummy_dir)
-
-        self.assertTrue(os.path.exists(dummy_file))
-        self.assertTrue(os.path.isdir(dummy_dir))
-
-        old_time = os.path.getmtime(c.src_clobber) - 60
-        os.utime(c.obj_clobber, (old_time, old_time))
-
-        self.assertTrue(c.clobber_needed())
-
-        required, performed, reason = c.maybe_do_clobber(c.topobjdir)
-        self.assertTrue(required)
-        self.assertTrue(performed)
-
-        self.assertFalse(os.path.exists(dummy_file))
-        self.assertFalse(os.path.exists(dummy_dir))
-
-    def test_cwd_under_topobjdir(self):
-        """If cwd is under topobjdir, we can't clobber."""
-
-        c = Clobberer(self.get_topsrcdir(), self.get_tempdir())
-
-        with open(c.obj_clobber, 'a'):
-            pass
-
-        old_time = os.path.getmtime(c.src_clobber) - 60
-        os.utime(c.obj_clobber, (old_time, old_time))
-
-        d = os.path.join(c.topobjdir, 'dummy_dir')
-        os.mkdir(d)
-
-        required, performed, reason = c.maybe_do_clobber(d)
-        self.assertTrue(required)
-        self.assertFalse(performed)
-        self.assertIn('Cannot clobber while the shell is inside', reason)
-
-
-    def test_mozconfig_overrides_auto_clobber(self):
-        """If NO_AUTOCLOBBER is in the environment, don't auto clobber."""
-
-        topsrcdir = self.get_topsrcdir()
-        topobjdir = self.get_tempdir()
-
-        obj_clobber = os.path.join(topobjdir, 'CLOBBER')
-        with open(obj_clobber, 'a'):
-            pass
-
-        dummy_file = os.path.join(topobjdir, 'dummy_file')
-        with open(dummy_file, 'a'):
-            pass
-
-        self.assertTrue(os.path.exists(dummy_file))
-
-        old_time = os.path.getmtime(os.path.join(topsrcdir, 'CLOBBER')) - 60
-        os.utime(obj_clobber, (old_time, old_time))
-
-        env = dict(os.environ)
-        env['NO_AUTOCLOBBER'] = '1'
-
-        s = StringIO()
-        status = clobber([topsrcdir, topobjdir], env, os.getcwd(), s)
-        self.assertEqual(status, 1)
-        self.assertIn('Automatic clobbering has been disabled', s.getvalue())
-        self.assertTrue(os.path.exists(dummy_file))
-
-
-if __name__ == '__main__':
-    main()