Bug 1416052 - Check clobber state from Python; r=nalexander
authorGregory Szorc <gps@mozilla.com>
Thu, 09 Nov 2017 23:13:59 -0800
changeset 391866 b6adf66f34c68b759ebdf468963464d4b2edb853
parent 391865 51d998c76d9afd42c1bda46373bb4a076f0fb0b7
child 391867 2a90d49c5573711b4c1f56558147eef124ca9131
push id97352
push usernerli@mozilla.com
push dateWed, 15 Nov 2017 10:16:19 +0000
treeherdermozilla-inbound@76d469663b3c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander
bugs1416052
milestone59.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1416052 - Check clobber state from Python; r=nalexander The clobber logic is already written in Python. Now that we always use mach in front of client.mk, we can check the clobber state before we execute client.mk. Since we always check the clobber state, we can remove the CLOBBER files from various dependencies in client.mk. The clobberer code should ensure everything is in a good state. The refactor of the clobber Python code required some changes to its testing. We drop some support for verifying output strings. But testing this correctly would require a bit of effort. I don't think it is worth it. MozReview-Commit-ID: 69CoImCgtNm
client.mk
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/controller/clobber.py
python/mozbuild/mozbuild/test/controller/test_clobber.py
--- a/client.mk
+++ b/client.mk
@@ -53,20 +53,16 @@ MOZCONFIG_CONTENT := $(subst ||,$(CR),$(
 include $(OBJDIR)/.mozconfig-client-mk
 
 # As '||' was used as a newline separator, it means it's not occurring in
 # lines themselves. It can thus safely be used to replaces normal spaces,
 # to then replace newlines with normal spaces. This allows to get a list
 # of mozconfig output lines.
 MOZCONFIG_OUT_LINES := $(subst $(CR), ,$(subst $(NULL) $(NULL),||,$(MOZCONFIG_CONTENT)))
 
-ifdef AUTOCLOBBER
-export AUTOCLOBBER=1
-endif
-
 ifdef MOZ_PARALLEL_BUILD
   MOZ_MAKE_FLAGS := $(filter-out -j%,$(MOZ_MAKE_FLAGS))
   MOZ_MAKE_FLAGS += -j$(MOZ_PARALLEL_BUILD)
 endif
 
 # Automatically add -jN to make flags if not defined. N defaults to number of cores.
 ifeq (,$(findstring -j,$(MOZ_MAKE_FLAGS)))
   cores=$(shell $(PYTHON) -c 'import multiprocessing; print(multiprocessing.cpu_count())')
@@ -96,17 +92,17 @@ build::
 
 ifndef MACH
 $(error client.mk must be used via `mach`. Try running \
 `./mach $(firstword $(MAKECMDGOALS) $(.DEFAULT_GOAL))`)
 endif
 
 # For now, only output "export" lines and lines containing UPLOAD_EXTRA_FILES.
 MOZCONFIG_MK_LINES := $(filter export||% UPLOAD_EXTRA_FILES% %UPLOAD_EXTRA_FILES%,$(MOZCONFIG_OUT_LINES))
-$(OBJDIR)/.mozconfig.mk: $(TOPSRCDIR)/client.mk $(FOUND_MOZCONFIG) $(OBJDIR)/CLOBBER
+$(OBJDIR)/.mozconfig.mk: $(TOPSRCDIR)/client.mk $(FOUND_MOZCONFIG)
 	$(if $(MOZCONFIG_MK_LINES),( $(foreach line,$(MOZCONFIG_MK_LINES), echo '$(subst ||, ,$(line))';) )) > $@
 
 # Include that makefile so that it is created. This should not actually change
 # the environment since MOZCONFIG_CONTENT, which MOZCONFIG_OUT_LINES derives
 # from, has already been eval'ed.
 include $(OBJDIR)/.mozconfig.mk
 
 # Print out any options loaded from mozconfig.
@@ -148,17 +144,16 @@ EXTRA_CONFIG_DEPS := \
 $(CONFIGURES): %: %.in $(EXTRA_CONFIG_DEPS)
 	@echo Generating $@
 	cp -f $< $@
 	chmod +x $@
 
 CONFIG_STATUS_DEPS := \
   $(wildcard $(TOPSRCDIR)/*/confvars.sh) \
   $(CONFIGURES) \
-  $(TOPSRCDIR)/CLOBBER \
   $(TOPSRCDIR)/nsprpub/configure \
   $(TOPSRCDIR)/config/milestone.txt \
   $(TOPSRCDIR)/browser/config/version.txt \
   $(TOPSRCDIR)/browser/config/version_display.txt \
   $(TOPSRCDIR)/build/virtualenv_packages.txt \
   $(TOPSRCDIR)/python/mozbuild/mozbuild/virtualenv.py \
   $(TOPSRCDIR)/testing/mozbase/packages.txt \
   $(OBJDIR)/.mozconfig.json \
@@ -176,24 +171,19 @@ CONFIGURE_ENV_ARGS += \
 #   $(TOPSRCDIR) will set @srcdir@ to "."; otherwise, it is set to the full
 #   path of $(TOPSRCDIR).
 ifeq ($(TOPSRCDIR),$(OBJDIR))
   CONFIGURE = ./configure
 else
   CONFIGURE = $(TOPSRCDIR)/configure
 endif
 
-$(OBJDIR)/CLOBBER: $(TOPSRCDIR)/CLOBBER
-	$(PYTHON) $(TOPSRCDIR)/config/pythonpath.py -I $(TOPSRCDIR)/testing/mozbase/mozfile \
-	    $(TOPSRCDIR)/python/mozbuild/mozbuild/controller/clobber.py $(TOPSRCDIR) $(OBJDIR)
-
 configure-files: $(CONFIGURES)
 
 configure-preqs = \
-  $(OBJDIR)/CLOBBER \
   configure-files \
   save-mozconfig \
   $(OBJDIR)/.mozconfig.json \
   $(NULL)
 
 CREATE_MOZCONFIG_JSON = $(shell $(TOPSRCDIR)/mach environment --format=json -o $(OBJDIR)/.mozconfig.json)
 # Force CREATE_MOZCONFIG_JSON above to be resolved, without side effects in
 # case the result is non empty, and allowing an override on the make command
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -1,15 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, unicode_literals
 
 import getpass
+import io
 import json
 import logging
 import os
 import subprocess
 import sys
 import time
 import which
 
@@ -27,16 +28,19 @@ try:
 except Exception:
     psutil = None
 
 from mach.mixin.logging import LoggingMixin
 from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
 
 import mozpack.path as mozpath
 
+from .clobber import (
+    Clobberer,
+)
 from ..base import (
     BuildEnvironmentNotFoundException,
     MozbuildObject,
 )
 from ..backend import (
     get_backend_class,
 )
 from ..testing import (
@@ -1324,16 +1328,20 @@ class BuildDriver(MozbuildObject):
     def _run_client_mk(self, target=None, line_handler=None, jobs=0,
                        verbose=None, keep_going=False, append_env=None):
         append_env = dict(append_env or {})
         append_env['TOPSRCDIR'] = self.topsrcdir
 
         append_env['CONFIG_GUESS'] = self.resolve_config_guess()
 
         mozconfig = self.mozconfig
+
+        if self._check_clobber(mozconfig, os.environ):
+            return 1
+
         mozconfig_client_mk = os.path.join(self.topobjdir,
                                            '.mozconfig-client-mk')
         with FileAvoidWrite(mozconfig_client_mk) as fh:
             for arg in mozconfig['make_extra'] or []:
                 fh.write(arg)
                 fh.write(b'\n')
             if mozconfig['make_flags']:
                 fh.write(b'MOZ_MAKE_FLAGS=%s\n' % b' '.join(mozconfig['make_flags']))
@@ -1354,8 +1362,37 @@ class BuildDriver(MozbuildObject):
                               print_directory=False,
                               target=target,
                               line_handler=line_handler,
                               log=False,
                               num_jobs=jobs,
                               silent=not verbose,
                               keep_going=keep_going,
                               append_env=append_env)
+
+    def _check_clobber(self, mozconfig, env):
+        auto_clobber = any([
+            env.get('AUTOCLOBBER', False),
+            (mozconfig['env'] or {}).get('added', {}).get('AUTOCLOBBER', False),
+            'AUTOCLOBBER=1' in (mozconfig['make_extra'] or []),
+        ])
+
+        clobberer = Clobberer(self.topsrcdir, self.topobjdir)
+        clobber_output = io.BytesIO()
+        res = clobberer.maybe_do_clobber(os.getcwd(), auto_clobber,
+                                         clobber_output)
+        clobber_output.seek(0)
+        for line in clobber_output.readlines():
+            self.log(logging.WARNING, 'clobber',
+                     {'msg': line.rstrip()}, '{msg}')
+
+        clobber_required, clobber_performed, clobber_message = res
+        if not clobber_required or clobber_performed:
+            if clobber_performed and env.get('TINDERBOX_OUTPUT'):
+                self.log(logging.WARNING, 'clobber',
+                         {'msg': 'TinderboxPrint: auto clobber'}, '{msg}')
+        else:
+            for line in clobber_message.splitlines():
+                self.log(logging.WARNING, 'clobber',
+                         {'msg': line.rstrip()}, '{msg}')
+            return True
+
+        return False
--- a/python/mozbuild/mozbuild/controller/clobber.py
+++ b/python/mozbuild/mozbuild/controller/clobber.py
@@ -199,39 +199,8 @@ class Clobberer(object):
             return True, False, self._message(
                 'Error when automatically clobbering: ' + str(error))
 
     def _message(self, reason):
         lines = [' ' + line for line in self.clobber_cause()]
 
         return CLOBBER_MESSAGE.format(clobber_reason='\n'.join(lines),
             no_reason='  ' + reason, clobber_file=self.obj_clobber)
-
-
-def main(args, env, cwd, fh=sys.stderr):
-    if len(args) != 2:
-        print('Usage: clobber.py topsrcdir topobjdir', file=fh)
-        return 1
-
-    topsrcdir, topobjdir = args
-
-    if not os.path.isabs(topsrcdir):
-        topsrcdir = os.path.abspath(topsrcdir)
-
-    if not os.path.isabs(topobjdir):
-        topobjdir = os.path.abspath(topobjdir)
-
-    auto = True if env.get('AUTOCLOBBER', False) else False
-    clobber = Clobberer(topsrcdir, topobjdir)
-    required, performed, message = clobber.maybe_do_clobber(cwd, auto, fh)
-
-    if not required or performed:
-        if performed and env.get('TINDERBOX_OUTPUT'):
-            print('TinderboxPrint: auto clobber', file=fh)
-        return 0
-
-    print(message, file=fh)
-    return 1
-
-
-if __name__ == '__main__':
-    sys.exit(main(sys.argv[1:], os.environ, os.getcwd(), sys.stdout))
-
--- a/python/mozbuild/mozbuild/test/controller/test_clobber.py
+++ b/python/mozbuild/mozbuild/test/controller/test_clobber.py
@@ -8,18 +8,25 @@ import os
 import shutil
 import tempfile
 import unittest
 
 from StringIO import StringIO
 
 from mozunit import main
 
-from mozbuild.controller.clobber import Clobberer
-from mozbuild.controller.clobber import main as clobber
+from mozbuild.base import (
+    MozbuildObject,
+)
+from mozbuild.controller.building import (
+    BuildDriver,
+)
+from mozbuild.controller.clobber import (
+    Clobberer,
+)
 
 
 class TestClobberer(unittest.TestCase):
     def setUp(self):
         self._temp_dirs = []
 
         return unittest.TestCase.setUp(self)
 
@@ -188,26 +195,26 @@ class TestClobberer(unittest.TestCase):
         old_time = os.path.getmtime(os.path.join(topsrcdir, 'CLOBBER')) - 60
         os.utime(obj_clobber, (old_time, old_time))
 
         # Check auto clobber is off by default
         env = dict(os.environ)
         if env.get('AUTOCLOBBER', False):
             del env['AUTOCLOBBER']
 
-        s = StringIO()
-        status = clobber([topsrcdir, topobjdir], env, os.getcwd(), s)
-        self.assertEqual(status, 1)
-        self.assertIn('Automatic clobbering is not enabled', s.getvalue())
+        mbo = MozbuildObject(topsrcdir, None, None, topobjdir)
+        build = mbo._spawn(BuildDriver)
+
+        status = build._check_clobber(build.mozconfig, env)
+
+        self.assertEqual(status, True)
         self.assertTrue(os.path.exists(dummy_file))
 
         # Check auto clobber opt-in works
         env['AUTOCLOBBER'] = '1'
 
-        s = StringIO()
-        status = clobber([topsrcdir, topobjdir], env, os.getcwd(), s)
-        self.assertEqual(status, 0)
-        self.assertIn('Successfully completed auto clobber', s.getvalue())
+        status = build._check_clobber(build.mozconfig, env)
+        self.assertFalse(status)
         self.assertFalse(os.path.exists(dummy_file))
 
 
 if __name__ == '__main__':
     main()