author Manish Goregaokar <>
Tue, 07 May 2019 02:34:53 +0000
changeset 531649 7aee5a30dd15cf0e203705808de4fc84cd56393d
parent 527620 426c8e0063416e851eb09473580fdd4962436911
child 533231 e3a0fedff65ea47dba3607665a1d34241d0c39c4
permissions -rw-r--r--
Bug 1549636: use uuid4() in mach; r=glandium Differential Revision:

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at

# This module provides functionality for the command-line build tool
# (mach). It is packaged as a module because everything is a library.

from __future__ import absolute_import, print_function, unicode_literals
from collections import Iterable

import argparse
import codecs
import errno
import imp
import logging
import os
import sys
import traceback
import uuid

from .base import (

from .decorators import (

from .config import ConfigSettings
from .dispatcher import CommandAction
from .logging import LoggingManager
from .registrar import Registrar

You can invoke |./mach busted| to check if this issue is already on file. If it
isn't, please use |./mach busted file| to report it. If |./mach busted| is
misbehaving, you can also inspect the dependencies of bug 1543241.

The error occurred in mach itself. This is likely a bug in mach itself or a
fundamental problem with a loaded module.

'''.lstrip() + SUGGEST_MACH_BUSTED

If filing a bug, please include the full output of mach, including this error

The details of the failure are as follows:

The error occurred in the implementation of the invoked mach command.

This should never occur and is likely a bug in the implementation of that
'''.lstrip() + SUGGEST_MACH_BUSTED

The error occurred in code that was called by the mach command. This is either
a bug in the called code itself or in the way that mach is calling it.
'''.lstrip() + SUGGEST_MACH_BUSTED

It looks like you tried to run mach without a command.

Run |mach help| to show a list of commands.

It looks like you are trying to %s an unknown mach command: %s
Run |mach help| to show a list of commands.

Did you want to %s any of these commands instead: %s?

It looks like you passed an unrecognized argument into mach.

The %s command does not accept the arguments: %s

Entry points should return a list of command providers or directories
containing command providers. The following entry point is invalid:


You are seeing this because there is an error in an external module attempting
to implement a mach command. Please fix the error, or uninstall the module from
your system.

class ArgumentParser(argparse.ArgumentParser):
    """Custom implementation argument parser to make things look pretty."""

    def error(self, message):
        """Custom error reporter to give more helpful text on bad commands."""
        if not message.startswith('argument command: invalid choice'):
            argparse.ArgumentParser.error(self, message)
            assert False

        print('Invalid command specified. The list of commands is below.\n')

    def format_help(self):
        text = argparse.ArgumentParser.format_help(self)

        # Strip out the silly command list that would preceed the pretty list.
        # Commands:
        #   {foo,bar}
        #     foo  Do foo.
        #     bar  Do bar.
        search = 'Commands:\n  {'
        start = text.find(search)

        if start != -1:
            end = text.find('}\n', start)
            assert end != -1

            real_start = start + len('Commands:\n')
            real_end = end + len('}\n')

            text = text[0:real_start] + text[real_end:]

        return text

class ContextWrapper(object):
    def __init__(self, context, handler):
        object.__setattr__(self, '_context', context)
        object.__setattr__(self, '_handler', handler)

    def __getattribute__(self, key):
            return getattr(object.__getattribute__(self, '_context'), key)
        except AttributeError as e:
                ret = object.__getattribute__(self, '_handler')(self, key)
            except (AttributeError, TypeError):
                # TypeError is in case the handler comes from old code not
                # taking a key argument.
                raise e
            setattr(self, key, ret)
            return ret

    def __setattr__(self, key, value):
        setattr(object.__getattribute__(self, '_context'), key, value)

class Mach(object):
    """Main mach driver type.

    This type is responsible for holding global mach state and dispatching
    a command from arguments.

    The following attributes may be assigned to the instance to influence

        populate_context_handler -- If defined, it must be a callable. The
            callable signature is the following:
                populate_context_handler(context, key=None)
            It acts as a fallback getter for the mach.base.CommandContext
            This allows to augment the context instance with arbitrary data
            for use in command handlers.
            For backwards compatibility, it is also called before command
            dispatch without a key, allowing the context handler to add
            attributes to the context instance.

        require_conditions -- If True, commands that do not have any condition
            functions applied will be skipped. Defaults to False.

        settings_paths -- A list of files or directories in which to search
            for settings files to load.


    USAGE = """%(prog)s [global arguments] command [command arguments]

mach (German for "do") is the main interface to the Mozilla build system and
common developer tasks.

You tell mach the command you want to perform and it does it for you.

Some common commands are:

    %(prog)s build     Build/compile the source tree.
    %(prog)s help      Show full help, including the list of all commands.

To see more help for a specific command, run:

  %(prog)s help <command>

    def __init__(self, cwd):
        assert os.path.isdir(cwd)

        self.cwd = cwd
        self.log_manager = LoggingManager()
        self.logger = logging.getLogger(__name__)
        self.settings = ConfigSettings()
        self.settings_paths = []

        if 'MACHRC' in os.environ:

        self.global_arguments = []
        self.populate_context_handler = None

    def add_global_argument(self, *args, **kwargs):
        """Register a global argument with the argument parser.

        Arguments are proxied to ArgumentParser.add_argument()

        self.global_arguments.append((args, kwargs))

    def load_commands_from_directory(self, path):
        """Scan for mach commands from modules in a directory.

        This takes a path to a directory, loads the .py files in it, and
        registers and found mach command providers with this mach instance.
        for f in sorted(os.listdir(path)):
            if not f.endswith('.py') or f == '':

            full_path = os.path.join(path, f)
            module_name = 'mach.commands.%s' % f[0:-3]

            self.load_commands_from_file(full_path, module_name=module_name)

    def load_commands_from_file(self, path, module_name=None):
        """Scan for mach commands from a file.

        This takes a path to a file and loads it as a Python module under the
        module name specified. If no name is specified, a random one will be
        if module_name is None:
            # Ensure parent module is present otherwise we'll (likely) get
            # an error due to unknown parent.
            if b'mach.commands' not in sys.modules:
                mod = imp.new_module(b'mach.commands')
                sys.modules[b'mach.commands'] = mod

            module_name = 'mach.commands.%s' % uuid.uuid4().get_hex()

            imp.load_source(module_name, path)
        except IOError as e:
            if e.errno != errno.ENOENT:

            raise MissingFileError('%s does not exist' % path)

    def load_commands_from_entry_point(self, group='mach.providers'):
        """Scan installed packages for mach command provider entry points. An
        entry point is a function that returns a list of paths to files or
        directories containing command providers.

        This takes an optional group argument which specifies the entry point
        group to use. If not specified, it defaults to 'mach.providers'.
            import pkg_resources
        except ImportError:
            print("Could not find setuptools, ignoring command entry points",

        for entry in pkg_resources.iter_entry_points(group=group, name=None):
            paths = entry.load()()
            if not isinstance(paths, Iterable):
                print(INVALID_ENTRY_POINT % entry)

            for path in paths:
                if os.path.isfile(path):
                elif os.path.isdir(path):
                    print("command provider '%s' does not exist" % path)

    def define_category(self, name, title, description, priority=50):
        """Provide a description for a named command category."""

        Registrar.register_category(name, title, description, priority)

    def require_conditions(self):
        return Registrar.require_conditions

    def require_conditions(self, value):
        Registrar.require_conditions = value

    def run(self, argv, stdin=None, stdout=None, stderr=None):
        """Runs mach with arguments provided from the command line.

        Returns the integer exit code that should be used. 0 means success. All
        other values indicate failure.

        # If no encoding is defined, we default to UTF-8 because without this
        # Python 2.7 will assume the default encoding of ASCII. This will blow
        # up with UnicodeEncodeError as soon as it encounters a non-ASCII
        # character in a unicode instance. We simply install a wrapper around
        # the streams and restore once we have finished.
        stdin = sys.stdin if stdin is None else stdin
        stdout = sys.stdout if stdout is None else stdout
        stderr = sys.stderr if stderr is None else stderr

        orig_stdin = sys.stdin
        orig_stdout = sys.stdout
        orig_stderr = sys.stderr

        sys.stdin = stdin
        sys.stdout = stdout
        sys.stderr = stderr

        orig_env = dict(os.environ)

            if stdin.encoding is None:
                sys.stdin = codecs.getreader('utf-8')(stdin)

            if stdout.encoding is None:
                sys.stdout = codecs.getwriter('utf-8')(stdout)

            if stderr.encoding is None:
                sys.stderr = codecs.getwriter('utf-8')(stderr)

            # Allow invoked processes (which may not have a handle on the
            # original stdout file descriptor) to know if the original stdout
            # is a TTY. This provides a mechanism to allow said processes to
            # enable emitting code codes, for example.
            if os.isatty(orig_stdout.fileno()):
                os.environ[b'MACH_STDOUT_ISATTY'] = b'1'

            return self._run(argv)
        except KeyboardInterrupt:
            print('mach interrupted by signal or user action. Stopping.')
            return 1

        except Exception:
            # _run swallows exceptions in invoked handlers and converts them to
            # a proper exit code. So, the only scenario where we should get an
            # exception here is if _run itself raises. If _run raises, that's a
            # bug in mach (or a loaded command module being silly) and thus
            # should be reported differently.
            self._print_error_header(argv, sys.stdout)

            exc_type, exc_value, exc_tb = sys.exc_info()
            stack = traceback.extract_tb(exc_tb)

            self._print_exception(sys.stdout, exc_type, exc_value, stack)

            return 1


            sys.stdin = orig_stdin
            sys.stdout = orig_stdout
            sys.stderr = orig_stderr

    def _run(self, argv):
        # Load settings as early as possible so things in
        # can use them.
        for provider in Registrar.settings_providers:

        context = CommandContext(cwd=self.cwd,
                                 settings=self.settings, log_manager=self.log_manager,

        if self.populate_context_handler:
            context = ContextWrapper(context, self.populate_context_handler)

        parser = self.get_argument_parser(context)

        if not len(argv):
            # We don't register the usage until here because if it is globally
            # registered, argparse always prints it. This is not desired when
            # running with --help.
            parser.usage = Mach.USAGE
            return 0

            args = parser.parse_args(argv)
        except NoCommandError:
            return 1
        except UnknownCommandError as e:
            suggestion_message = SUGGESTED_COMMANDS_MESSAGE % (
                e.verb, ', '.join(e.suggested_commands)) if e.suggested_commands else ''
            print(UNKNOWN_COMMAND_ERROR %
                  (e.verb, e.command, suggestion_message))
            return 1
        except UnrecognizedArgumentError as e:
            print(UNRECOGNIZED_ARGUMENT_ERROR % (e.command,
                                                 ' '.join(e.arguments)))
            return 1

        # Add JSON logging to a file if requested.
        if args.logfile:

        # Up the logging level if requested.
        log_level = logging.INFO
        if args.verbose:
            log_level = logging.DEBUG


        write_times = True
        if args.log_no_times or 'MACH_NO_WRITE_TIMES' in os.environ:
            write_times = False

        # Always enable terminal logging. The log manager figures out if we are
        # actually in a TTY or are a pipe and does the right thing.

        if args.settings_file:
            # Argument parsing has already happened, so settings that apply
            # to command line handling (e.g alias, defaults) will be ignored.

        def _check_debugger(program):
            """Checks if debugger specified in command line is installed.

            Uses mozdebug to locate debuggers.

            If the call does not raise any exceptions, mach is permitted
            to continue execution.

            Otherwise, mach execution is halted.

                program (str): debugger program name.
            import mozdebug
            info = mozdebug.get_debugger_info(program)
            if info is None:
                print("Specified debugger '{}' is not found.\n".format(program) +
                      "Is it installed? Is it in your PATH?")

        # For the codepath where ./mach <test_type> --debugger=<program>,
        # assert that debugger value exists first, then check if installed on system.
        if (hasattr(args.command_args, "debugger") and
                getattr(args.command_args, "debugger") is not None):
            _check_debugger(getattr(args.command_args, "debugger"))
        # For the codepath where ./mach test --debugger=<program> <test_type>,
        # debugger must be specified from command line with the = operator.
        # Otherwise, an IndexError is raised, which is converted to an exit code of 1.
        elif (hasattr(args.command_args, "extra_args") and
                getattr(args.command_args, "extra_args")):
            extra_args = getattr(args.command_args, "extra_args")
                debugger = [ea.split("=")[1] for ea in extra_args if "debugger" in ea]
            except IndexError:
                print("Debugger must be specified with '=' when invoking ./mach test.\n" +
                      "Please correct the command and try again.")
            if debugger:

        if not hasattr(args, 'mach_handler'):
            raise MachError('ArgumentParser result missing mach handler info.')

        handler = getattr(args, 'mach_handler')

        # if --disable-tests flag was enabled in the mozconfig used to compile
        # the build, tests will be disabled.
        # instead of trying to run nonexistent tests then reporting a failure,
        # this will prevent mach from progressing beyond this point.
        if handler.category == 'testing':
            from mozbuild.base import BuildEnvironmentNotFoundException
                from mozbuild.base import MozbuildObject
                # all environments should have an instance of build object.
                build = MozbuildObject.from_environment()
                if build is not None and hasattr(build, 'mozconfig'):
                    ac_options = build.mozconfig['configure_args']
                    if ac_options and '--disable-tests' in ac_options:
                        print('Tests have been disabled by mozconfig with the flag' +
                              '"ac_add_options --disable-tests".\n' +
                              'Remove the flag, and re-compile to enable tests.')
                        return 1
            except BuildEnvironmentNotFoundException:
                # likely automation environment, so do nothing.

            return Registrar._run_command_handler(handler, context=context,
        except KeyboardInterrupt as ki:
            raise ki
        except FailedCommandError as e:
            return e.exit_code
        except Exception as e:
            exc_type, exc_value, exc_tb = sys.exc_info()

            # The first two frames are us and are never used.
            stack = traceback.extract_tb(exc_tb)[2:]

            # If we have nothing on the stack, the exception was raised as part
            # of calling the @Command method itself. This likely means a
            # mismatch between @CommandArgument and arguments to the method.
            # e.g. there exists a @CommandArgument without the corresponding
            # argument on the method. We handle that here until the module
            # loader grows the ability to validate better.
            if not len(stack):
                self._print_exception(sys.stdout, exc_type, exc_value,
                return 1

            # Split the frames into those from the module containing the
            # command and everything else.
            command_frames = []
            other_frames = []

            initial_file = stack[0][0]

            for frame in stack:
                if frame[0] == initial_file:

            # If the exception was in the module providing the command, it's
            # likely the bug is in the mach command module, not something else.
            # If there are other frames, the bug is likely not the mach
            # command's fault.
            self._print_error_header(argv, sys.stdout)

            if len(other_frames):

            self._print_exception(sys.stdout, exc_type, exc_value, stack)

            return 1

    def log(self, level, action, params, format_str):
        """Helper method to record a structured log event."""
        self.logger.log(level, format_str,
                        extra={'action': action, 'params': params})

    def _print_error_header(self, argv, fh):
        fh.write('Error running mach:\n\n')
        fh.write('    ')

    def _print_exception(self, fh, exc_type, exc_value, stack):

        for l in traceback.format_exception_only(exc_type, exc_value):

        for l in traceback.format_list(stack):

    def load_settings(self, paths):
        """Load the specified settings files.

        If a directory is specified, the following basenames will be
        searched for in this order:

            machrc, .machrc
        if isinstance(paths, basestring):
            paths = [paths]

        valid_names = ('machrc', '.machrc')

        def find_in_dir(base):
            if os.path.isfile(base):
                return base

            for name in valid_names:
                path = os.path.join(base, name)
                if os.path.isfile(path):
                    return path

        files = map(find_in_dir, self.settings_paths)
        files = filter(bool, files)


    def get_argument_parser(self, context):
        """Returns an argument parser for the command-line interface."""

        parser = ArgumentParser(add_help=False,
                                usage='%(prog)s [global arguments] '
                                'command [command arguments]')

        # Order is important here as it dictates the order the auto-generated
        # help messages are printed.
        global_group = parser.add_argument_group('Global Arguments')

        global_group.add_argument('-v', '--verbose', dest='verbose',
                                  action='store_true', default=False,
                                  help='Print verbose output.')
        global_group.add_argument('-l', '--log-file', dest='logfile',
                                  metavar='FILENAME', type=argparse.FileType('ab'),
                                  help='Filename to write log data to.')
        global_group.add_argument('--log-interval', dest='log_interval',
                                  action='store_true', default=False,
                                  help='Prefix log line with interval from last message rather '
                                  'than relative time. Note that this is NOT execution time '
                                  'if there are parallel operations.')
        suppress_log_by_default = False
        if 'INSIDE_EMACS' in os.environ:
            suppress_log_by_default = True
        global_group.add_argument('--log-no-times', dest='log_no_times',
                                  action='store_true', default=suppress_log_by_default,
                                  help='Do not prefix log lines with times. By default, '
                                  'mach will prefix each output line with the time since '
                                  'command start.')
        global_group.add_argument('-h', '--help', dest='help',
                                  action='store_true', default=False,
                                  help='Show this help message.')
        global_group.add_argument('--debug-command', action='store_true',
                                  help='Start a Python debugger when command is dispatched.')
        global_group.add_argument('--settings', dest='settings_file',
                                  metavar='FILENAME', default=None,
                                  help='Path to settings file.')

        for args, kwargs in self.global_arguments:
            global_group.add_argument(*args, **kwargs)

        # We need to be last because CommandAction swallows all remaining
        # arguments and argparse parses arguments in the order they were added.
        parser.add_argument('command', action=CommandAction,
                            registrar=Registrar, context=context)

        return parser