python/mozbuild/mozbuild/controller/clobber.py
author Gregory Szorc <gps@mozilla.com>
Wed, 15 Nov 2017 10:17:26 -0800
changeset 391942 62b191e02145b1220a39dd2034a3f4d7057e133f
parent 391817 b6adf66f34c68b759ebdf468963464d4b2edb853
child 474128 2715bac40d2f2e90a60cb86a28308bfeb4fc34de
permissions -rw-r--r--
Bug 1417387 - Don't mix unicode and bytes in clobber code; r=nalexander b6adf66f34c6 (bug 1416052) changed the value for "fh" when this code is called. It can now be an io.BytesIO. This type enforces that arguments are bytes and doesn't perform automatic type coercion like most other parts of Python 2. self.topobjdir is a unicode. And unicode_literals isn't in effect in this file. So convert self.topobjdir to bytes to make BytesIO happy. MozReview-Commit-ID: LrWTKFp3ZKT

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from __future__ import absolute_import, print_function

r'''This module contains code for managing clobbering of the tree.'''

import errno
import os
import subprocess
import sys

from mozfile.mozfile import remove as mozfileremove
from textwrap import TextWrapper


CLOBBER_MESSAGE = ''.join([TextWrapper().fill(line) + '\n' for line in
'''
The CLOBBER file has been updated, indicating that an incremental build since \
your last build will probably not work. A full/clobber build is required.

The reason for the clobber is:

{clobber_reason}

Clobbering can be performed automatically. However, we didn't automatically \
clobber this time because:

{no_reason}

The easiest and fastest way to clobber is to run:

 $ mach clobber

If you know this clobber doesn't apply to you or you're feeling lucky -- \
Well, are ya? -- you can ignore this clobber requirement by running:

 $ touch {clobber_file}
'''.splitlines()])

class Clobberer(object):
    def __init__(self, topsrcdir, topobjdir):
        """Create a new object to manage clobbering the tree.

        It is bound to a top source directory and to a specific object
        directory.
        """
        assert os.path.isabs(topsrcdir)
        assert os.path.isabs(topobjdir)

        self.topsrcdir = os.path.normpath(topsrcdir)
        self.topobjdir = os.path.normpath(topobjdir)
        self.src_clobber = os.path.join(topsrcdir, 'CLOBBER')
        self.obj_clobber = os.path.join(topobjdir, 'CLOBBER')

        # Try looking for mozilla/CLOBBER, for comm-central
        if not os.path.isfile(self.src_clobber):
            self.src_clobber = os.path.join(topsrcdir, 'mozilla', 'CLOBBER')

        assert os.path.isfile(self.src_clobber)

    def clobber_needed(self):
        """Returns a bool indicating whether a tree clobber is required."""

        # No object directory clobber file means we're good.
        if not os.path.exists(self.obj_clobber):
            return False

        # Object directory clobber older than current is fine.
        if os.path.getmtime(self.src_clobber) <= \
            os.path.getmtime(self.obj_clobber):

            return False

        return True

    def clobber_cause(self):
        """Obtain the cause why a clobber is required.

        This reads the cause from the CLOBBER file.

        This returns a list of lines describing why the clobber was required.
        Each line is stripped of leading and trailing whitespace.
        """
        with open(self.src_clobber, 'rt') as fh:
            lines = [l.strip() for l in fh.readlines()]
            return [l for l in lines if l and not l.startswith('#')]

    def have_winrm(self):
        # `winrm -h` should print 'winrm version ...' and exit 1
        try:
            p = subprocess.Popen(['winrm.exe', '-h'],
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT)
            return p.wait() == 1 and p.stdout.read().startswith('winrm')
        except:
            return False

    def remove_objdir(self, full=True):
        """Remove the object directory.

        ``full`` controls whether to fully delete the objdir. If False,
        some directories (e.g. Visual Studio Project Files) will not be
        deleted.
        """
        # Top-level files and directories to not clobber by default.
        no_clobber = {
            '.mozbuild',
            'msvc',
        }

        if full:
            # mozfile doesn't like unicode arguments (bug 818783).
            paths = [self.topobjdir.encode('utf-8')]
        else:
            try:
                paths = []
                for p in os.listdir(self.topobjdir):
                    if p not in no_clobber:
                        paths.append(os.path.join(self.topobjdir, p).encode('utf-8'))
            except OSError as e:
                if e.errno != errno.ENOENT:
                    raise
                return

        procs = []
        for p in sorted(paths):
            path = os.path.join(self.topobjdir, p)
            if sys.platform.startswith('win') and self.have_winrm() and os.path.isdir(path):
                procs.append(subprocess.Popen(['winrm', '-rf', path]))
            else:
                # We use mozfile because it is faster than shutil.rmtree().
                mozfileremove(path)

        for p in procs:
            p.wait()

    def ensure_objdir_state(self):
        """Ensure the CLOBBER file in the objdir exists.

        This is called as part of the build to ensure the clobber information
        is configured properly for the objdir.
        """
        if not os.path.exists(self.topobjdir):
            os.makedirs(self.topobjdir)

        if not os.path.exists(self.obj_clobber):
            # Simply touch the file.
            with open(self.obj_clobber, 'a'):
                pass

    def maybe_do_clobber(self, cwd, allow_auto=False, fh=sys.stderr):
        """Perform a clobber if it is required. Maybe.

        This is the API the build system invokes to determine if a clobber
        is needed and to automatically perform that clobber if we can.

        This returns a tuple of (bool, bool, str). The elements are:

          - Whether a clobber was/is required.
          - Whether a clobber was performed.
          - The reason why the clobber failed or could not be performed. This
            will be None if no clobber is required or if we clobbered without
            error.
        """
        assert cwd
        cwd = os.path.normpath(cwd)

        if not self.clobber_needed():
            print('Clobber not needed.', file=fh)
            self.ensure_objdir_state()
            return False, False, None

        # So a clobber is needed. We only perform a clobber if we are
        # allowed to perform an automatic clobber (off by default) and if the
        # current directory is not under the object directory. The latter is
        # because operating systems, filesystems, and shell can throw fits
        # if the current working directory is deleted from under you. While it
        # can work in some scenarios, we take the conservative approach and
        # never try.
        if not allow_auto:
            return True, False, \
               self._message('Automatic clobbering is not enabled\n'
                              '  (add "mk_add_options AUTOCLOBBER=1" to your '
                              'mozconfig).')

        if cwd.startswith(self.topobjdir) and cwd != self.topobjdir:
            return True, False, self._message(
                'Cannot clobber while the shell is inside the object directory.')

        objdir = self.topobjdir.encode('utf-8', 'replace')
        print('Automatically clobbering %s' % objdir, file=fh)
        try:
            self.remove_objdir(False)
            self.ensure_objdir_state()
            print('Successfully completed auto clobber.', file=fh)
            return True, True, None
        except (IOError) as error:
            return True, False, self._message(
                'Error when automatically clobbering: ' + str(error))

    def _message(self, reason):
        lines = [' ' + line for line in self.clobber_cause()]

        return CLOBBER_MESSAGE.format(clobber_reason='\n'.join(lines),
            no_reason='  ' + reason, clobber_file=self.obj_clobber)