author | Gregory Szorc <gps@mozilla.com> |
Fri, 29 Mar 2013 10:34:58 -0700 | |
changeset 126719 | a61aa08ab4ac92a5f909a87f0aa717a05eb36df2 |
parent 126718 | 88ec6ee2d57a2c62051e13aba2c83bc071ccc47c |
child 126720 | dc0b6744aef665c2445664d3758fa828a28e3d8a |
push id | 25609 |
push user | gszorc@mozilla.com |
push date | Fri, 29 Mar 2013 17:37:03 +0000 |
treeherder | mozilla-inbound@a61aa08ab4ac [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | ted |
bugs | 837323 |
milestone | 22.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/Makefile.in +++ b/Makefile.in @@ -30,24 +30,30 @@ include $(topsrcdir)/config/config.mk GARBAGE_DIRS += dist _javagen _profile _tests staticlib DIST_GARBAGE = config.cache config.log config.status* config-defs.h \ config/autoconf.mk \ unallmakefiles mozilla-config.h \ netwerk/necko-config.h xpcom/xpcom-config.h xpcom/xpcom-private.h \ $(topsrcdir)/.mozconfig.mk $(topsrcdir)/.mozconfig.out -default alldep all:: $(topsrcdir)/configure config.status +default alldep all:: CLOBBER $(topsrcdir)/configure config.status $(RM) -r $(DIST)/sdk $(RM) -r $(DIST)/include $(RM) -r $(DIST)/private $(RM) -r $(DIST)/public $(RM) $(DIST)/bin/chrome.manifest $(DIST)/bin/components/components.manifest $(RM) -r _tests +CLOBBER: $(topsrcdir)/CLOBBER + @echo "STOP! The CLOBBER file has changed." + @echo "Please run the build through a sanctioned build wrapper, such as" + @echo "'mach build' or client.mk." + @exit 1 + $(topsrcdir)/configure: $(topsrcdir)/configure.in @echo "STOP! configure.in has changed, and your configure is out of date." @echo "Please rerun autoconf and re-configure your build directory." @echo "To ignore this message, touch 'configure' in the source directory," @echo "but your build might not succeed." @exit 1 config.status: $(topsrcdir)/configure
--- a/client.mk +++ b/client.mk @@ -103,16 +103,19 @@ define CR endef # As $(shell) doesn't preserve newlines, use sed to replace them with an # unlikely sequence (||), which is then replaced back to newlines by make # before evaluation. $(eval $(subst ||,$(CR),$(shell _PYMAKE=$(.PYMAKE) $(TOPSRCDIR)/$(MOZCONFIG_LOADER) $(TOPSRCDIR) 2> $(TOPSRCDIR)/.mozconfig.out | sed 's/$$/||/'))) +ifdef NO_AUTOCLOBBER +export NO_AUTOCLOBBER=1 +endif # Automatically add -jN to make flags if not defined. N defaults to number of cores. ifeq (,$(findstring -j,$(MOZ_MAKE_FLAGS))) cores=$(shell $(PYTHON) -c 'import multiprocessing; print(multiprocessing.cpu_count())') MOZ_MAKE_FLAGS += -j$(cores) endif @@ -297,19 +300,23 @@ CONFIGURE_ENV_ARGS += \ # $(TOPSRCDIR) will set @srcdir@ to "."; otherwise, it is set to the full # path of $(TOPSRCDIR). ifeq ($(TOPSRCDIR),$(OBJDIR)) CONFIGURE = ./configure else CONFIGURE = $(TOPSRCDIR)/configure endif +check-clobber: + $(PYTHON) $(TOPSRCDIR)/python/mozbuild/mozbuild/controller/clobber.py $(TOPSRCDIR) $(OBJDIR) + configure-files: $(CONFIGURES) configure-preqs = \ + check-clobber \ configure-files \ $(call mkdir_deps,$(OBJDIR)) \ $(if $(MOZ_BUILD_PROJECTS),$(call mkdir_deps,$(MOZ_OBJDIR))) \ save-mozconfig \ $(NULL) save-mozconfig: $(FOUND_MOZCONFIG) -cp $(FOUND_MOZCONFIG) $(OBJDIR)/.mozconfig @@ -437,9 +444,28 @@ check-sync-dirs-%: echo-variable-%: @echo $($*) # This makefile doesn't support parallel execution. It does pass # MOZ_MAKE_FLAGS to sub-make processes, so they will correctly execute # in parallel. .NOTPARALLEL: -.PHONY: checkout real_checkout depend realbuild build profiledbuild cleansrcdir pull_all build_all clobber clobber_all pull_and_build_all everything configure preflight_all preflight postflight postflight_all $(OBJDIR_TARGETS) +.PHONY: checkout \ + real_checkout \ + depend \ + realbuild \ + build \ + profiledbuild \ + cleansrcdir \ + pull_all \ + build_all \ + check-clobber \ + clobber \ + clobber_all \ + pull_and_build_all \ + everything \ + configure \ + preflight_all \ + preflight \ + postflight \ + postflight_all \ + $(OBJDIR_TARGETS)
--- a/configure.in +++ b/configure.in @@ -115,40 +115,16 @@ then *** EOF exit 1 break fi fi MOZ_BUILD_ROOT=`pwd` -dnl Do not allow building if a clobber is required -dnl ============================================================== -dnl TODO Make this better, ideally this would clobber automaticially -if test -e $_objdir/CLOBBER; then - if test $_topsrcdir/CLOBBER -nt $_objdir/CLOBBER; then - echo " ***" - echo " * The CLOBBER file has been updated, indicating that an incremental build" - echo " * since your last build will probably not work. A full build is required." - echo " * The change that caused this is:" - cat $_topsrcdir/CLOBBER | sed '/^#/d' | sed 's/^/ * /' - echo " * " - echo " * The easiest way to fix this is to manually delete your objdir:" - echo " * rm -rf $_objdir" - echo " * " - echo " * Or, if you know this clobber doesn't apply to you, it can be ignored with:" - echo " * cp '$_topsrcdir/CLOBBER' $_objdir" - echo " ***" - exit 1 - break; - fi -else - touch $_objdir/CLOBBER -fi - MOZ_PYTHON MOZ_DEFAULT_COMPILER COMPILE_ENVIRONMENT=1 MOZ_ARG_DISABLE_BOOL(compile-environment, [ --disable-compile-environment Disable compiler/library checks.],
--- a/python/Makefile.in +++ b/python/Makefile.in @@ -7,16 +7,17 @@ topsrcdir := @top_srcdir@ srcdir := @srcdir@ VPATH = @srcdir@ include $(DEPTH)/config/autoconf.mk test_dirs := \ mozbuild/mozbuild/test \ mozbuild/mozbuild/test/backend \ + mozbuild/mozbuild/test/controller \ mozbuild/mozbuild/test/compilation \ mozbuild/mozbuild/test/frontend \ mozbuild/mozpack/test \ $(NULL) PYTHON_UNIT_TESTS := $(foreach dir,$(test_dirs),$(wildcard $(srcdir)/$(dir)/*.py)) include $(topsrcdir)/config/rules.mk
new file mode 100644 --- /dev/null +++ b/python/mozbuild/mozbuild/controller/clobber.py @@ -0,0 +1,194 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import print_function + +r'''This module contains code for managing clobbering of the tree.''' + +import os +import sys + +try: + from mozfile.mozfile import rmtree +except ImportError: + from shutil import rmtree + + +CLOBBER_MESSAGE = ''' +*** +* The CLOBBER file has been updated, indicating that an incremental build since +* your last build will probably not work. A full/clobber build is required. +* +* The reason for the clobber is: +* +{clobber_reason} +* +* Clobbering can be performed automatically. However, we didn't automatically +* clobber this time because: +* +* {no_reason} +* +* The easiest and fastest way to clobber is to run: +* +* $ mach clobber +* +* If you know this clobber doesn't apply to you or you're feeling lucky - well +* do ya? - you can ignore this clobber requirement by running: +* +* $ touch {clobber_file} +* +*** +'''.strip() + + +class Clobberer(object): + def __init__(self, topsrcdir, topobjdir): + """Create a new object to manage clobbering the tree. + + It is bound to a top source directory and to a specific object + directory. + """ + assert os.path.isabs(topsrcdir) + assert os.path.isabs(topobjdir) + + self.topsrcdir = os.path.normpath(topsrcdir) + self.topobjdir = os.path.normpath(topobjdir) + self.src_clobber = os.path.join(topsrcdir, 'CLOBBER') + self.obj_clobber = os.path.join(topobjdir, 'CLOBBER') + + assert os.path.isfile(self.src_clobber) + + def clobber_needed(self): + """Returns a bool indicating whether a tree clobber is required.""" + + # No object directory clobber file means we're good. + if not os.path.exists(self.obj_clobber): + return False + + # Object directory clobber older than current is fine. + if os.path.getmtime(self.src_clobber) <= \ + os.path.getmtime(self.obj_clobber): + + return False + + return True + + def clobber_cause(self): + """Obtain the cause why a clobber is required. + + This reads the cause from the CLOBBER file. + + This returns a list of lines describing why the clobber was required. + Each line is stripped of leading and trailing whitespace. + """ + with open(self.src_clobber, 'rt') as fh: + lines = [l.strip() for l in fh.readlines()] + return [l for l in lines if l and not l.startswith('#')] + + def ensure_objdir_state(self): + """Ensure the CLOBBER file in the objdir exists. + + This is called as part of the build to ensure the clobber information + is configured properly for the objdir. + """ + if not os.path.exists(self.topobjdir): + os.makedirs(self.topobjdir) + + if not os.path.exists(self.obj_clobber): + # Simply touch the file. + with open(self.obj_clobber, 'a'): + pass + + def maybe_do_clobber(self, cwd, allow_auto=True, fh=sys.stderr): + """Perform a clobber if it is required. Maybe. + + This is the API the build system invokes to determine if a clobber + is needed and to automatically perform that clobber if we can. + + This returns a tuple of (bool, bool, str). The elements are: + + - Whether a clobber was/is required. + - Whether a clobber was performed. + - The reason why the clobber failed or could not be performed. This + will be None if no clobber is required or if we clobbered without + error. + """ + assert cwd + cwd = os.path.normpath(cwd) + + if not self.clobber_needed(): + print('Clobber not needed.', file=fh) + self.ensure_objdir_state() + return False, False, None + + # So a clobber is needed. We only perform a clobber if we are + # allowed to perform an automatic clobber (the default) and if the + # current directory is not under the object directory. The latter is + # because operating systems, filesystems, and shell can throw fits + # if the current working directory is deleted from under you. While it + # can work in some scenarios, we take the conservative approach and + # never try. + if not allow_auto: + return True, False, self._message( + 'Automatic clobbering has been disabled.') + + if cwd.startswith(self.topobjdir) and cwd != self.topobjdir: + return True, False, self._message( + 'Cannot clobber while the shell is inside the object directory.') + + print('Automatically clobbering %s' % self.topobjdir, file=fh) + try: + if cwd == self.topobjdir: + for entry in os.listdir(self.topobjdir): + full = os.path.join(self.topobjdir, entry) + + if os.path.isdir(full): + rmtree(full) + else: + os.unlink(full) + + else: + rmtree(self.topobjdir) + + self.ensure_objdir_state() + print('Successfully completed auto clobber.', file=fh) + return True, True, None + except (IOError) as error: + return True, False, self._message( + 'Error when automatically clobbering: ' + str(error)) + + def _message(self, reason): + lines = ['* ' + line for line in self.clobber_cause()] + + return CLOBBER_MESSAGE.format(clobber_reason='\n'.join(lines), + no_reason=reason, clobber_file=self.obj_clobber) + + +def main(args, env, cwd, fh=sys.stderr): + if len(args) != 2: + print('Usage: clobber.py topsrcdir topobjdir', file=fh) + return 1 + + topsrcdir, topobjdir = args + + if not os.path.isabs(topsrcdir): + topsrcdir = os.path.abspath(topsrcdir) + + if not os.path.isabs(topobjdir): + topobjdir = os.path.abspath(topobjdir) + + auto = False if env.get('NO_AUTOCLOBBER', False) else True + clobber = Clobberer(topsrcdir, topobjdir) + required, performed, message = clobber.maybe_do_clobber(cwd, auto, fh) + + if not required or performed: + return 0 + + print(message, file=fh) + return 1 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:], os.environ, os.getcwd(), sys.stdout)) +
new file mode 100644 --- /dev/null +++ b/python/mozbuild/mozbuild/test/controller/test_clobber.py @@ -0,0 +1,208 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os +import shutil +import tempfile +import unittest + +from StringIO import StringIO + +from mozunit import main + +from mozbuild.controller.clobber import Clobberer +from mozbuild.controller.clobber import main as clobber + + +class TestClobberer(unittest.TestCase): + def setUp(self): + self._temp_dirs = [] + + return unittest.TestCase.setUp(self) + + def tearDown(self): + for d in self._temp_dirs: + shutil.rmtree(d, ignore_errors=True) + + return unittest.TestCase.tearDown(self) + + def get_tempdir(self): + t = tempfile.mkdtemp() + self._temp_dirs.append(t) + return t + + def get_topsrcdir(self): + t = self.get_tempdir() + p = os.path.join(t, 'CLOBBER') + with open(p, 'a'): + pass + + return t + + def test_no_objdir(self): + """If topobjdir does not exist, no clobber is needed.""" + + tmp = os.path.join(self.get_tempdir(), 'topobjdir') + self.assertFalse(os.path.exists(tmp)) + + c = Clobberer(self.get_topsrcdir(), tmp) + self.assertFalse(c.clobber_needed()) + + # Side-effect is topobjdir is created with CLOBBER file touched. + required, performed, reason = c.maybe_do_clobber(os.getcwd()) + self.assertFalse(required) + self.assertFalse(performed) + self.assertIsNone(reason) + + self.assertTrue(os.path.isdir(tmp)) + self.assertTrue(os.path.exists(os.path.join(tmp, 'CLOBBER'))) + + def test_objdir_no_clobber_file(self): + """If CLOBBER does not exist in topobjdir, treat as empty.""" + + c = Clobberer(self.get_topsrcdir(), self.get_tempdir()) + self.assertFalse(c.clobber_needed()) + + required, performed, reason = c.maybe_do_clobber(os.getcwd()) + self.assertFalse(required) + self.assertFalse(performed) + self.assertIsNone(reason) + + self.assertTrue(os.path.exists(os.path.join(c.topobjdir, 'CLOBBER'))) + + def test_objdir_clobber_newer(self): + """If CLOBBER in topobjdir is newer, do nothing.""" + + c = Clobberer(self.get_topsrcdir(), self.get_tempdir()) + with open(c.obj_clobber, 'a'): + pass + + required, performed, reason = c.maybe_do_clobber(os.getcwd()) + self.assertFalse(required) + self.assertFalse(performed) + self.assertIsNone(reason) + + def test_objdir_clobber_older(self): + """If CLOBBER in topobjdir is older, we clobber.""" + + c = Clobberer(self.get_topsrcdir(), self.get_tempdir()) + with open(c.obj_clobber, 'a'): + pass + + dummy_path = os.path.join(c.topobjdir, 'foo') + with open(dummy_path, 'a'): + pass + + self.assertTrue(os.path.exists(dummy_path)) + + old_time = os.path.getmtime(c.src_clobber) - 60 + os.utime(c.obj_clobber, (old_time, old_time)) + + self.assertTrue(c.clobber_needed()) + + required, performed, reason = c.maybe_do_clobber(os.getcwd(), False) + self.assertTrue(required) + self.assertFalse(performed) + self.assertIn('Automatic clobbering has been disabled', reason) + + # Now let's actually do it. + required, performed, reason = c.maybe_do_clobber(os.getcwd()) + self.assertTrue(required) + self.assertTrue(performed) + + self.assertFalse(os.path.exists(dummy_path)) + self.assertTrue(os.path.exists(c.obj_clobber)) + self.assertGreaterEqual(os.path.getmtime(c.obj_clobber), + os.path.getmtime(c.src_clobber)) + + def test_objdir_is_srcdir(self): + """If topobjdir is the topsrcdir, refuse to clobber.""" + + tmp = self.get_topsrcdir() + c = Clobberer(tmp, tmp) + + self.assertFalse(c.clobber_needed()) + + def test_cwd_is_topobjdir(self): + """If cwd is topobjdir, we can still clobber.""" + c = Clobberer(self.get_topsrcdir(), self.get_tempdir()) + + with open(c.obj_clobber, 'a'): + pass + + dummy_file = os.path.join(c.topobjdir, 'dummy_file') + with open(dummy_file, 'a'): + pass + + dummy_dir = os.path.join(c.topobjdir, 'dummy_dir') + os.mkdir(dummy_dir) + + self.assertTrue(os.path.exists(dummy_file)) + self.assertTrue(os.path.isdir(dummy_dir)) + + old_time = os.path.getmtime(c.src_clobber) - 60 + os.utime(c.obj_clobber, (old_time, old_time)) + + self.assertTrue(c.clobber_needed()) + + required, performed, reason = c.maybe_do_clobber(c.topobjdir) + self.assertTrue(required) + self.assertTrue(performed) + + self.assertFalse(os.path.exists(dummy_file)) + self.assertFalse(os.path.exists(dummy_dir)) + + def test_cwd_under_topobjdir(self): + """If cwd is under topobjdir, we can't clobber.""" + + c = Clobberer(self.get_topsrcdir(), self.get_tempdir()) + + with open(c.obj_clobber, 'a'): + pass + + old_time = os.path.getmtime(c.src_clobber) - 60 + os.utime(c.obj_clobber, (old_time, old_time)) + + d = os.path.join(c.topobjdir, 'dummy_dir') + os.mkdir(d) + + required, performed, reason = c.maybe_do_clobber(d) + self.assertTrue(required) + self.assertFalse(performed) + self.assertIn('Cannot clobber while the shell is inside', reason) + + + def test_mozconfig_overrides_auto_clobber(self): + """If NO_AUTOCLOBBER is in the environment, don't auto clobber.""" + + topsrcdir = self.get_topsrcdir() + topobjdir = self.get_tempdir() + + obj_clobber = os.path.join(topobjdir, 'CLOBBER') + with open(obj_clobber, 'a'): + pass + + dummy_file = os.path.join(topobjdir, 'dummy_file') + with open(dummy_file, 'a'): + pass + + self.assertTrue(os.path.exists(dummy_file)) + + old_time = os.path.getmtime(os.path.join(topsrcdir, 'CLOBBER')) - 60 + os.utime(obj_clobber, (old_time, old_time)) + + env = dict(os.environ) + env['NO_AUTOCLOBBER'] = '1' + + s = StringIO() + status = clobber([topsrcdir, topobjdir], env, os.getcwd(), s) + self.assertEqual(status, 1) + self.assertIn('Automatic clobbering has been disabled', s.getvalue()) + self.assertTrue(os.path.exists(dummy_file)) + + +if __name__ == '__main__': + main()