Backed out changesets 206bb159d62f, a6ab3e11f721, and a61aa08ab4ac (bug 837323) for frequent Windows fatal clobber errors on a CLOSED TREE.
authorRyan VanderMeulen <ryanvm@gmail.com>
Mon, 08 Apr 2013 09:59:45 -0400
changeset 128346 27211859cb34dd4bd369d45c5274d734a09b255e
parent 128345 b8078621fb2a6e8502fd8fd134737482453d30e6
child 128347 e19ace7a61e4817da259b5a2a9ca928b5033ae8a
push id270
push userpvanderbeken@mozilla.com
push dateThu, 06 Mar 2014 09:24:21 +0000
bugs837323
milestone23.0a1
backs out206bb159d62ffb02f39640cb7c4ec68903f27356
a6ab3e11f721e49e0b2499ba0fa54ecf80e35522
a61aa08ab4ac92a5f909a87f0aa717a05eb36df2
Backed out changesets 206bb159d62f, a6ab3e11f721, and a61aa08ab4ac (bug 837323) for frequent Windows fatal clobber errors on a CLOSED TREE.
Makefile.in
client.mk
configure.in
python/Makefile.in
python/mozbuild/mozbuild/controller/__init__.py
python/mozbuild/mozbuild/controller/clobber.py
python/mozbuild/mozbuild/test/controller/__init__.py
python/mozbuild/mozbuild/test/controller/test_clobber.py
--- a/Makefile.in
+++ b/Makefile.in
@@ -30,30 +30,24 @@ include $(topsrcdir)/config/config.mk
 
 GARBAGE_DIRS += dist _javagen _profile _tests staticlib
 DIST_GARBAGE = config.cache config.log config.status* config-defs.h \
    config/autoconf.mk \
    unallmakefiles mozilla-config.h \
    netwerk/necko-config.h xpcom/xpcom-config.h xpcom/xpcom-private.h \
    $(topsrcdir)/.mozconfig.mk $(topsrcdir)/.mozconfig.out
 
-default alldep all:: CLOBBER $(topsrcdir)/configure config.status
+default alldep all:: $(topsrcdir)/configure config.status
 	$(RM) -r $(DIST)/sdk
 	$(RM) -r $(DIST)/include
 	$(RM) -r $(DIST)/private
 	$(RM) -r $(DIST)/public
 	$(RM) $(DIST)/bin/chrome.manifest $(DIST)/bin/components/components.manifest
 	$(RM) -r _tests
 
-CLOBBER: $(topsrcdir)/CLOBBER
-	@echo "STOP!  The CLOBBER file has changed."
-	@echo "Please run the build through a sanctioned build wrapper, such as"
-	@echo "'mach build' or client.mk."
-	@exit 1
-
 $(topsrcdir)/configure: $(topsrcdir)/configure.in
 	@echo "STOP!  configure.in has changed, and your configure is out of date."
 	@echo "Please rerun autoconf and re-configure your build directory."
 	@echo "To ignore this message, touch 'configure' in the source directory,"
 	@echo "but your build might not succeed."
 	@exit 1
 
 config.status: $(topsrcdir)/configure
--- a/client.mk
+++ b/client.mk
@@ -103,19 +103,16 @@ define CR
 
 endef
 
 # As $(shell) doesn't preserve newlines, use sed to replace them with an
 # unlikely sequence (||), which is then replaced back to newlines by make
 # before evaluation.
 $(eval $(subst ||,$(CR),$(shell _PYMAKE=$(.PYMAKE) $(TOPSRCDIR)/$(MOZCONFIG_LOADER) $(TOPSRCDIR) 2> $(TOPSRCDIR)/.mozconfig.out | sed 's/$$/||/')))
 
-ifdef NO_AUTOCLOBBER
-export NO_AUTOCLOBBER=1
-endif
 
 # Automatically add -jN to make flags if not defined. N defaults to number of cores.
 ifeq (,$(findstring -j,$(MOZ_MAKE_FLAGS)))
   cores=$(shell $(PYTHON) -c 'import multiprocessing; print(multiprocessing.cpu_count())')
   MOZ_MAKE_FLAGS += -j$(cores)
 endif
 
 
@@ -300,23 +297,19 @@ CONFIGURE_ENV_ARGS += \
 #   $(TOPSRCDIR) will set @srcdir@ to "."; otherwise, it is set to the full
 #   path of $(TOPSRCDIR).
 ifeq ($(TOPSRCDIR),$(OBJDIR))
   CONFIGURE = ./configure
 else
   CONFIGURE = $(TOPSRCDIR)/configure
 endif
 
-check-clobber:
-	$(PYTHON) $(TOPSRCDIR)/python/mozbuild/mozbuild/controller/clobber.py $(TOPSRCDIR) $(OBJDIR)
-
 configure-files: $(CONFIGURES)
 
 configure-preqs = \
-  check-clobber \
   configure-files \
   $(call mkdir_deps,$(OBJDIR)) \
   $(if $(MOZ_BUILD_PROJECTS),$(call mkdir_deps,$(MOZ_OBJDIR))) \
   save-mozconfig \
   $(NULL)
 
 save-mozconfig: $(FOUND_MOZCONFIG)
 	-cp $(FOUND_MOZCONFIG) $(OBJDIR)/.mozconfig
@@ -444,28 +437,9 @@ check-sync-dirs-%:
 echo-variable-%:
 	@echo $($*)
 
 # This makefile doesn't support parallel execution. It does pass
 # MOZ_MAKE_FLAGS to sub-make processes, so they will correctly execute
 # in parallel.
 .NOTPARALLEL:
 
-.PHONY: checkout \
-    real_checkout \
-    depend \
-    realbuild \
-    build \
-    profiledbuild \
-    cleansrcdir \
-    pull_all \
-    build_all \
-    check-clobber \
-    clobber \
-    clobber_all \
-    pull_and_build_all \
-    everything \
-    configure \
-    preflight_all \
-    preflight \
-    postflight \
-    postflight_all \
-    $(OBJDIR_TARGETS)
+.PHONY: checkout real_checkout depend realbuild build profiledbuild cleansrcdir pull_all build_all clobber clobber_all pull_and_build_all everything configure preflight_all preflight postflight postflight_all $(OBJDIR_TARGETS)
--- a/configure.in
+++ b/configure.in
@@ -115,16 +115,40 @@ then
 	***
 	EOF
     exit 1
     break
   fi
 fi
 MOZ_BUILD_ROOT=`pwd`
 
+dnl Do not allow building if a clobber is required
+dnl ==============================================================
+dnl TODO Make this better, ideally this would clobber automaticially
+if test -e $_objdir/CLOBBER; then
+  if test $_topsrcdir/CLOBBER -nt $_objdir/CLOBBER; then
+    echo "	***"
+    echo "	*	The CLOBBER file has been updated, indicating that an incremental build"
+    echo "	*	since your last build will probably not work. A full build is required."
+    echo "	*	The change that caused this is:"
+    cat $_topsrcdir/CLOBBER | sed '/^#/d' | sed 's/^/	*	/'
+    echo "	*	"
+    echo "	*	The easiest way to fix this is to manually delete your objdir:"
+    echo "	*	rm -rf $_objdir"
+    echo "	*	"
+    echo "	*	Or, if you know this clobber doesn't apply to you, it can be ignored with:"
+    echo "	*	cp '$_topsrcdir/CLOBBER' $_objdir"
+    echo "	***"
+    exit 1
+    break;
+  fi
+else
+  touch $_objdir/CLOBBER
+fi
+
 MOZ_PYTHON
 
 MOZ_DEFAULT_COMPILER
 
 COMPILE_ENVIRONMENT=1
 MOZ_ARG_DISABLE_BOOL(compile-environment,
 [  --disable-compile-environment
                           Disable compiler/library checks.],
--- a/python/Makefile.in
+++ b/python/Makefile.in
@@ -7,17 +7,16 @@ topsrcdir := @top_srcdir@
 srcdir := @srcdir@
 VPATH = @srcdir@
 
 include $(DEPTH)/config/autoconf.mk
 
 test_dirs := \
   mozbuild/mozbuild/test \
   mozbuild/mozbuild/test/backend \
-  mozbuild/mozbuild/test/controller \
   mozbuild/mozbuild/test/compilation \
   mozbuild/mozbuild/test/frontend \
   mozbuild/mozpack/test \
   $(NULL)
 
 PYTHON_UNIT_TESTS := $(foreach dir,$(test_dirs),$(wildcard $(srcdir)/$(dir)/*.py))
 
 include $(topsrcdir)/config/rules.mk
deleted file mode 100644
deleted file mode 100644
--- a/python/mozbuild/mozbuild/controller/clobber.py
+++ /dev/null
@@ -1,194 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import print_function
-
-r'''This module contains code for managing clobbering of the tree.'''
-
-import os
-import sys
-
-try:
-    from mozfile.mozfile import rmtree
-except ImportError:
-    from shutil import rmtree
-
-
-CLOBBER_MESSAGE = '''
-***
-* The CLOBBER file has been updated, indicating that an incremental build since
-* your last build will probably not work. A full/clobber build is required.
-*
-* The reason for the clobber is:
-*
-{clobber_reason}
-*
-* Clobbering can be performed automatically. However, we didn't automatically
-* clobber this time because:
-*
-*   {no_reason}
-*
-* The easiest and fastest way to clobber is to run:
-*
-*  $ mach clobber
-*
-* If you know this clobber doesn't apply to you or you're feeling lucky - well
-* do ya? - you can ignore this clobber requirement by running:
-*
-*  $ touch {clobber_file}
-*
-***
-'''.strip()
-
-
-class Clobberer(object):
-    def __init__(self, topsrcdir, topobjdir):
-        """Create a new object to manage clobbering the tree.
-
-        It is bound to a top source directory and to a specific object
-        directory.
-        """
-        assert os.path.isabs(topsrcdir)
-        assert os.path.isabs(topobjdir)
-
-        self.topsrcdir = os.path.normpath(topsrcdir)
-        self.topobjdir = os.path.normpath(topobjdir)
-        self.src_clobber = os.path.join(topsrcdir, 'CLOBBER')
-        self.obj_clobber = os.path.join(topobjdir, 'CLOBBER')
-
-        assert os.path.isfile(self.src_clobber)
-
-    def clobber_needed(self):
-        """Returns a bool indicating whether a tree clobber is required."""
-
-        # No object directory clobber file means we're good.
-        if not os.path.exists(self.obj_clobber):
-            return False
-
-        # Object directory clobber older than current is fine.
-        if os.path.getmtime(self.src_clobber) <= \
-            os.path.getmtime(self.obj_clobber):
-
-            return False
-
-        return True
-
-    def clobber_cause(self):
-        """Obtain the cause why a clobber is required.
-
-        This reads the cause from the CLOBBER file.
-
-        This returns a list of lines describing why the clobber was required.
-        Each line is stripped of leading and trailing whitespace.
-        """
-        with open(self.src_clobber, 'rt') as fh:
-            lines = [l.strip() for l in fh.readlines()]
-            return [l for l in lines if l and not l.startswith('#')]
-
-    def ensure_objdir_state(self):
-        """Ensure the CLOBBER file in the objdir exists.
-
-        This is called as part of the build to ensure the clobber information
-        is configured properly for the objdir.
-        """
-        if not os.path.exists(self.topobjdir):
-            os.makedirs(self.topobjdir)
-
-        if not os.path.exists(self.obj_clobber):
-            # Simply touch the file.
-            with open(self.obj_clobber, 'a'):
-                pass
-
-    def maybe_do_clobber(self, cwd, allow_auto=True, fh=sys.stderr):
-        """Perform a clobber if it is required. Maybe.
-
-        This is the API the build system invokes to determine if a clobber
-        is needed and to automatically perform that clobber if we can.
-
-        This returns a tuple of (bool, bool, str). The elements are:
-
-          - Whether a clobber was/is required.
-          - Whether a clobber was performed.
-          - The reason why the clobber failed or could not be performed. This
-            will be None if no clobber is required or if we clobbered without
-            error.
-        """
-        assert cwd
-        cwd = os.path.normpath(cwd)
-
-        if not self.clobber_needed():
-            print('Clobber not needed.', file=fh)
-            self.ensure_objdir_state()
-            return False, False, None
-
-        # So a clobber is needed. We only perform a clobber if we are
-        # allowed to perform an automatic clobber (the default) and if the
-        # current directory is not under the object directory. The latter is
-        # because operating systems, filesystems, and shell can throw fits
-        # if the current working directory is deleted from under you. While it
-        # can work in some scenarios, we take the conservative approach and
-        # never try.
-        if not allow_auto:
-            return True, False, self._message(
-                'Automatic clobbering has been disabled.')
-
-        if cwd.startswith(self.topobjdir) and cwd != self.topobjdir:
-            return True, False, self._message(
-                'Cannot clobber while the shell is inside the object directory.')
-
-        print('Automatically clobbering %s' % self.topobjdir, file=fh)
-        try:
-            if cwd == self.topobjdir:
-                for entry in os.listdir(self.topobjdir):
-                    full = os.path.join(self.topobjdir, entry)
-
-                    if os.path.isdir(full):
-                        rmtree(full)
-                    else:
-                        os.unlink(full)
-
-            else:
-                rmtree(self.topobjdir)
-
-            self.ensure_objdir_state()
-            print('Successfully completed auto clobber.', file=fh)
-            return True, True, None
-        except (IOError) as error:
-            return True, False, self._message(
-                'Error when automatically clobbering: ' + str(error))
-
-    def _message(self, reason):
-        lines = ['*  ' + line for line in self.clobber_cause()]
-
-        return CLOBBER_MESSAGE.format(clobber_reason='\n'.join(lines),
-            no_reason=reason, clobber_file=self.obj_clobber)
-
-
-def main(args, env, cwd, fh=sys.stderr):
-    if len(args) != 2:
-        print('Usage: clobber.py topsrcdir topobjdir', file=fh)
-        return 1
-
-    topsrcdir, topobjdir = args
-
-    if not os.path.isabs(topsrcdir):
-        topsrcdir = os.path.abspath(topsrcdir)
-
-    if not os.path.isabs(topobjdir):
-        topobjdir = os.path.abspath(topobjdir)
-
-    auto = False if env.get('NO_AUTOCLOBBER', False) else True
-    clobber = Clobberer(topsrcdir, topobjdir)
-    required, performed, message = clobber.maybe_do_clobber(cwd, auto, fh)
-
-    if not required or performed:
-        return 0
-
-    print(message, file=fh)
-    return 1
-
-
-if __name__ == '__main__':
-    sys.exit(main(sys.argv[1:], os.environ, os.getcwd(), sys.stdout))
-
deleted file mode 100644
deleted file mode 100644
--- a/python/mozbuild/mozbuild/test/controller/test_clobber.py
+++ /dev/null
@@ -1,208 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import unicode_literals
-
-import os
-import shutil
-import tempfile
-import unittest
-
-from StringIO import StringIO
-
-from mozunit import main
-
-from mozbuild.controller.clobber import Clobberer
-from mozbuild.controller.clobber import main as clobber
-
-
-class TestClobberer(unittest.TestCase):
-    def setUp(self):
-        self._temp_dirs = []
-
-        return unittest.TestCase.setUp(self)
-
-    def tearDown(self):
-        for d in self._temp_dirs:
-            shutil.rmtree(d, ignore_errors=True)
-
-        return unittest.TestCase.tearDown(self)
-
-    def get_tempdir(self):
-        t = tempfile.mkdtemp()
-        self._temp_dirs.append(t)
-        return t
-
-    def get_topsrcdir(self):
-        t = self.get_tempdir()
-        p = os.path.join(t, 'CLOBBER')
-        with open(p, 'a'):
-            pass
-
-        return t
-
-    def test_no_objdir(self):
-        """If topobjdir does not exist, no clobber is needed."""
-
-        tmp = os.path.join(self.get_tempdir(), 'topobjdir')
-        self.assertFalse(os.path.exists(tmp))
-
-        c = Clobberer(self.get_topsrcdir(), tmp)
-        self.assertFalse(c.clobber_needed())
-
-        # Side-effect is topobjdir is created with CLOBBER file touched.
-        required, performed, reason = c.maybe_do_clobber(os.getcwd())
-        self.assertFalse(required)
-        self.assertFalse(performed)
-        self.assertIsNone(reason)
-
-        self.assertTrue(os.path.isdir(tmp))
-        self.assertTrue(os.path.exists(os.path.join(tmp, 'CLOBBER')))
-
-    def test_objdir_no_clobber_file(self):
-        """If CLOBBER does not exist in topobjdir, treat as empty."""
-
-        c = Clobberer(self.get_topsrcdir(), self.get_tempdir())
-        self.assertFalse(c.clobber_needed())
-
-        required, performed, reason = c.maybe_do_clobber(os.getcwd())
-        self.assertFalse(required)
-        self.assertFalse(performed)
-        self.assertIsNone(reason)
-
-        self.assertTrue(os.path.exists(os.path.join(c.topobjdir, 'CLOBBER')))
-
-    def test_objdir_clobber_newer(self):
-        """If CLOBBER in topobjdir is newer, do nothing."""
-
-        c = Clobberer(self.get_topsrcdir(), self.get_tempdir())
-        with open(c.obj_clobber, 'a'):
-            pass
-
-        required, performed, reason = c.maybe_do_clobber(os.getcwd())
-        self.assertFalse(required)
-        self.assertFalse(performed)
-        self.assertIsNone(reason)
-
-    def test_objdir_clobber_older(self):
-        """If CLOBBER in topobjdir is older, we clobber."""
-
-        c = Clobberer(self.get_topsrcdir(), self.get_tempdir())
-        with open(c.obj_clobber, 'a'):
-            pass
-
-        dummy_path = os.path.join(c.topobjdir, 'foo')
-        with open(dummy_path, 'a'):
-            pass
-
-        self.assertTrue(os.path.exists(dummy_path))
-
-        old_time = os.path.getmtime(c.src_clobber) - 60
-        os.utime(c.obj_clobber, (old_time, old_time))
-
-        self.assertTrue(c.clobber_needed())
-
-        required, performed, reason = c.maybe_do_clobber(os.getcwd(), False)
-        self.assertTrue(required)
-        self.assertFalse(performed)
-        self.assertIn('Automatic clobbering has been disabled', reason)
-
-        # Now let's actually do it.
-        required, performed, reason = c.maybe_do_clobber(os.getcwd())
-        self.assertTrue(required)
-        self.assertTrue(performed)
-
-        self.assertFalse(os.path.exists(dummy_path))
-        self.assertTrue(os.path.exists(c.obj_clobber))
-        self.assertGreaterEqual(os.path.getmtime(c.obj_clobber),
-            os.path.getmtime(c.src_clobber))
-
-    def test_objdir_is_srcdir(self):
-        """If topobjdir is the topsrcdir, refuse to clobber."""
-
-        tmp = self.get_topsrcdir()
-        c = Clobberer(tmp, tmp)
-
-        self.assertFalse(c.clobber_needed())
-
-    def test_cwd_is_topobjdir(self):
-        """If cwd is topobjdir, we can still clobber."""
-        c = Clobberer(self.get_topsrcdir(), self.get_tempdir())
-
-        with open(c.obj_clobber, 'a'):
-            pass
-
-        dummy_file = os.path.join(c.topobjdir, 'dummy_file')
-        with open(dummy_file, 'a'):
-            pass
-
-        dummy_dir = os.path.join(c.topobjdir, 'dummy_dir')
-        os.mkdir(dummy_dir)
-
-        self.assertTrue(os.path.exists(dummy_file))
-        self.assertTrue(os.path.isdir(dummy_dir))
-
-        old_time = os.path.getmtime(c.src_clobber) - 60
-        os.utime(c.obj_clobber, (old_time, old_time))
-
-        self.assertTrue(c.clobber_needed())
-
-        required, performed, reason = c.maybe_do_clobber(c.topobjdir)
-        self.assertTrue(required)
-        self.assertTrue(performed)
-
-        self.assertFalse(os.path.exists(dummy_file))
-        self.assertFalse(os.path.exists(dummy_dir))
-
-    def test_cwd_under_topobjdir(self):
-        """If cwd is under topobjdir, we can't clobber."""
-
-        c = Clobberer(self.get_topsrcdir(), self.get_tempdir())
-
-        with open(c.obj_clobber, 'a'):
-            pass
-
-        old_time = os.path.getmtime(c.src_clobber) - 60
-        os.utime(c.obj_clobber, (old_time, old_time))
-
-        d = os.path.join(c.topobjdir, 'dummy_dir')
-        os.mkdir(d)
-
-        required, performed, reason = c.maybe_do_clobber(d)
-        self.assertTrue(required)
-        self.assertFalse(performed)
-        self.assertIn('Cannot clobber while the shell is inside', reason)
-
-
-    def test_mozconfig_overrides_auto_clobber(self):
-        """If NO_AUTOCLOBBER is in the environment, don't auto clobber."""
-
-        topsrcdir = self.get_topsrcdir()
-        topobjdir = self.get_tempdir()
-
-        obj_clobber = os.path.join(topobjdir, 'CLOBBER')
-        with open(obj_clobber, 'a'):
-            pass
-
-        dummy_file = os.path.join(topobjdir, 'dummy_file')
-        with open(dummy_file, 'a'):
-            pass
-
-        self.assertTrue(os.path.exists(dummy_file))
-
-        old_time = os.path.getmtime(os.path.join(topsrcdir, 'CLOBBER')) - 60
-        os.utime(obj_clobber, (old_time, old_time))
-
-        env = dict(os.environ)
-        env['NO_AUTOCLOBBER'] = '1'
-
-        s = StringIO()
-        status = clobber([topsrcdir, topobjdir], env, os.getcwd(), s)
-        self.assertEqual(status, 1)
-        self.assertIn('Automatic clobbering has been disabled', s.getvalue())
-        self.assertTrue(os.path.exists(dummy_file))
-
-
-if __name__ == '__main__':
-    main()