Bug 751795 - Part 1: mach, the new frontend to mozilla-central; r=jhammel
authorGregory Szorc <gps@mozilla.com>
Wed, 26 Sep 2012 09:43:54 -0700
changeset 114404 4621dc706abc54c668f15d83455c898f6007fbe2
parent 114403 9226912d54eb802b27ff8109286c4f1c44bfaff7
child 114405 c9294c9df7c1b2b04a04d333728c0956be18f7e9
push id1708
push userakeybl@mozilla.com
push dateMon, 19 Nov 2012 21:10:21 +0000
treeherdermozilla-beta@27b14fe50103 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjhammel
bugs751795
milestone18.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 751795 - Part 1: mach, the new frontend to mozilla-central; r=jhammel
.gitignore
.hgignore
mach
python/mach/README.rst
python/mach/mach/__init__.py
python/mach/mach/base.py
python/mach/mach/main.py
python/mach/mach/terminal.py
python/mach/setup.py
--- a/.gitignore
+++ b/.gitignore
@@ -14,16 +14,17 @@ ID
 
 # User files that may appear at the root
 /.mozconfig*
 /mozconfig
 /configure
 /config.cache
 /config.log
 /.clang_complete
+/mach.ini
 
 # Empty marker file that's generated when we check out NSS
 security/manager/.nss.checkout
 
 # Build directories
 /obj*/
 
 # Build directories for js shell
--- a/.hgignore
+++ b/.hgignore
@@ -13,16 +13,17 @@
 
 # User files that may appear at the root
 ^\.mozconfig
 ^mozconfig*
 ^configure$
 ^config\.cache$
 ^config\.log$
 ^\.clang_complete
+^mach.ini$
 
 # Empty marker file that's generated when we check out NSS
 ^security/manager/\.nss\.checkout$
 
 # Build directories
 ^obj
 
 # Build directories for js shell
new file mode 100755
--- /dev/null
+++ b/mach
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import os
+import platform
+import sys
+
+# Ensure we are running Python 2.7+. We put this check here so we generate a
+# user-friendly error message rather than a cryptic stack trace on module
+# import.
+if sys.version_info[0] == 2 and sys.version_info[1] < 7:
+    print('Python 2.7 or above is required to run mach.')
+    print('You are running', platform.python_version())
+    sys.exit(1)
+
+# TODO Bug 794506 Integrate with the in-tree virtualenv configuration.
+SEARCH_PATHS = [
+    'python/mach',
+    'python/mozbuild',
+    'build',
+    'build/pymake',
+    'python/blessings',
+    'python/psutil',
+    'python/which',
+    'other-licenses/ply',
+    'xpcom/idl-parser',
+    'testing/xpcshell',
+    'testing/mozbase/mozprocess',
+    'testing/mozbase/mozinfo',
+]
+
+our_dir = os.path.dirname(os.path.abspath(__file__))
+
+try:
+    import mach.main
+except ImportError:
+    SEARCH_PATHS.reverse()
+    sys.path[0:0] = [os.path.join(our_dir, path) for path in SEARCH_PATHS]
+
+    import mach.main
+
+# All of the code is in a module because EVERYTHING IS A LIBRARY.
+mach = mach.main.Mach(our_dir)
+mach.run(sys.argv[1:])
new file mode 100644
--- /dev/null
+++ b/python/mach/README.rst
@@ -0,0 +1,103 @@
+The mach Driver
+===============
+
+The *mach* driver is the command line interface (CLI) to the source tree.
+
+The *mach* driver is invoked by running the *mach* script or from
+instantiating the *Mach* class from the *mach.main* module.
+
+Implementing mach Commands
+--------------------------
+
+The *mach* driver follows the convention of popular tools like Git,
+Subversion, and Mercurial and provides a common driver for multiple
+sub-commands.
+
+Modules inside *mach* typically contain 1 or more classes which
+inherit from *mach.base.ArgumentProvider*. Modules that inherit from
+this class are hooked up to the *mach* CLI driver. So, to add a new
+sub-command/action to *mach*, one simply needs to create a new class in
+the *mach* package which inherits from *ArgumentProvider*.
+
+Currently, you also need to hook up some plumbing in
+*mach.main.Mach*. In the future, we hope to have automatic detection
+of submodules.
+
+Your command class performs the role of configuring the *mach* frontend
+argument parser as well as providing the methods invoked if a command is
+requested. These methods will take the user-supplied input, do something
+(likely by calling a backend function in a separate module), then format
+output to the terminal.
+
+The plumbing to hook up the arguments to the *mach* driver involves
+light magic. At *mach* invocation time, the driver creates a new
+*argparse* instance. For each registered class that provides commands,
+it calls the *populate_argparse* static method, passing it the parser
+instance.
+
+Your class's *populate_argparse* function should register sub-commands
+with the parser.
+
+For example, say you want to provide the *doitall* command. e.g. *mach
+doitall*. You would create the module *mach.doitall* and this
+module would contain the following class:
+
+    from mach.base import ArgumentProvider
+
+    class DoItAll(ArgumentProvider):
+        def run(self, more=False):
+            print 'I did it!'
+
+        @staticmethod
+        def populate_argparse(parser):
+            # Create the parser to handle the sub-command.
+            p = parser.add_parser('doitall', help='Do it all!')
+
+            p.add_argument('more', action='store_true', default=False,
+                help='Do more!')
+
+            # Tell driver that the handler for this sub-command is the
+            # method *run* on the class *DoItAll*.
+            p.set_defaults(cls=DoItAll, method='run')
+
+The most important line here is the call to *set_defaults*.
+Specifically, the *cls* and *method* parameters, which tell the driver
+which class to instantiate and which method to execute if this command
+is requested.
+
+The specified method will receive all arguments parsed from the command.
+It is important that you use named - not positional - arguments for your
+handler functions or things will blow up. This is because the mach driver
+is using the ``**kwargs`` notation to call the defined method.
+
+In the future, we may provide additional syntactical sugar to make all
+this easier. For example, we may provide decorators on methods to hook
+up commands and handlers.
+
+Minimizing Code in Mach
+-----------------------
+
+Mach is just a frontend. Therefore, code in this package should pertain to
+one of 3 areas:
+
+1. Obtaining user input (parsing arguments, prompting, etc)
+2. Calling into some other Python package
+3. Formatting output
+
+Mach should not contain core logic pertaining to the desired task. If you
+find yourself needing to invent some new functionality, you should implement
+it as a generic package outside of mach and then write a mach shim to call
+into it. There are many advantages to this approach, including reusability
+outside of mach (others may want to write other frontends) and easier testing
+(it is easier to test generic libraries than code that interacts with the
+command line or terminal).
+
+Keeping Frontend Modules Small
+------------------------------
+
+The frontend modules providing mach commands are currently all loaded when
+the mach CLI driver starts. Therefore, there is potential for *import bloat*.
+
+We want the CLI driver to load quickly. So, please delay load external modules
+until they are actually required. In other words, don't use a global
+*import* when you can import from inside a specific command's handler.
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/python/mach/mach/base.py
@@ -0,0 +1,13 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+
+class ArgumentProvider(object):
+    """Base class for classes wishing to provide CLI arguments to mach."""
+
+    @staticmethod
+    def populate_argparse(parser):
+        raise Exception("populate_argparse not implemented.")
new file mode 100644
--- /dev/null
+++ b/python/mach/mach/main.py
@@ -0,0 +1,193 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This module provides functionality for the command-line build tool
+# (mach). It is packaged as a module because everything is a library.
+
+from __future__ import unicode_literals
+
+import argparse
+import logging
+import os
+import sys
+
+from mozbuild.base import BuildConfig
+from mozbuild.config import ConfigSettings
+from mozbuild.logger import LoggingManager
+
+# Import sub-command modules
+# TODO Bug 794509 do this via auto-discovery. Update README once this is
+# done.
+# TODO import modules
+
+# Classes inheriting from ArgumentProvider that provide commands.
+HANDLERS = [
+]
+
+# Classes inheriting from ConfigProvider that provide settings.
+# TODO this should come from auto-discovery somehow.
+SETTINGS_PROVIDERS = [
+    BuildConfig,
+]
+
+# Settings for argument parser that don't get proxied to sub-module. i.e. these
+# are things consumed by the driver itself.
+CONSUMED_ARGUMENTS = [
+    'settings_file',
+    'verbose',
+    'logfile',
+    'log_interval',
+    'action',
+    'cls',
+    'method',
+    'func',
+]
+
+class Mach(object):
+    """Contains code for the command-line `mach` interface."""
+
+    USAGE = """%(prog)s subcommand [arguments]
+
+mach provides an interface to performing common developer tasks. You specify
+an action/sub-command and it performs it.
+
+Some common actions are:
+
+    %(prog)s help      Show full help, including the list of all commands.
+
+To see more help for a specific action, run:
+
+  %(prog)s <command> --help
+"""
+
+    def __init__(self, cwd):
+        assert os.path.isdir(cwd)
+
+        self.cwd = cwd
+        self.log_manager = LoggingManager()
+        self.logger = logging.getLogger(__name__)
+        self.settings = ConfigSettings()
+
+        self.log_manager.register_structured_logger(self.logger)
+
+    def run(self, argv):
+        """Runs mach with arguments provided from the command line."""
+        parser = self.get_argument_parser()
+
+        if not len(argv):
+            # We don't register the usage until here because if it is globally
+            # registered, argparse always prints it. This is not desired when
+            # running with --help.
+            parser.usage = Mach.USAGE
+            parser.print_usage()
+            return 0
+
+        if argv[0] == 'help':
+            parser.print_help()
+            return 0
+
+        args = parser.parse_args(argv)
+
+        # Add JSON logging to a file if requested.
+        if args.logfile:
+            self.log_manager.add_json_handler(args.logfile)
+
+        # Up the logging level if requested.
+        log_level = logging.INFO
+        if args.verbose:
+            log_level = logging.DEBUG
+
+        # Always enable terminal logging. The log manager figures out if we are
+        # actually in a TTY or are a pipe and does the right thing.
+        self.log_manager.add_terminal_logging(level=log_level,
+            write_interval=args.log_interval)
+
+        self.load_settings(args)
+        conf = BuildConfig(self.settings)
+
+        stripped = {k: getattr(args, k) for k in vars(args) if k not in
+            CONSUMED_ARGUMENTS}
+
+        # If the action is associated with a class, instantiate and run it.
+        # All classes must be Base-derived and take the expected argument list.
+        if hasattr(args, 'cls'):
+            cls = getattr(args, 'cls')
+            instance = cls(self.cwd, self.settings, self.log_manager)
+            fn = getattr(instance, getattr(args, 'method'))
+
+        # If the action is associated with a function, call it.
+        elif hasattr(args, 'func'):
+            fn = getattr(args, 'func')
+        else:
+            raise Exception('Dispatch configuration error in module.')
+
+        fn(**stripped)
+
+    def log(self, level, action, params, format_str):
+        """Helper method to record a structured log event."""
+        self.logger.log(level, format_str,
+            extra={'action': action, 'params': params})
+
+    def load_settings(self, args):
+        """Determine which settings files apply and load them.
+
+        Currently, we only support loading settings from a single file.
+        Ideally, we support loading from multiple files. This is supported by
+        the ConfigSettings API. However, that API currently doesn't track where
+        individual values come from, so if we load from multiple sources then
+        save, we effectively do a full copy. We don't want this. Until
+        ConfigSettings does the right thing, we shouldn't expose multi-file
+        loading.
+
+        We look for a settings file in the following locations. The first one
+        found wins:
+
+          1) Command line argument
+          2) Environment variable
+          3) Default path
+        """
+        for provider in SETTINGS_PROVIDERS:
+            provider.register_settings()
+            self.settings.register_provider(provider)
+
+        p = os.path.join(self.cwd, 'mach.ini')
+
+        if args.settings_file:
+            p = args.settings_file
+        elif 'MACH_SETTINGS_FILE' in os.environ:
+            p = os.environ['MACH_SETTINGS_FILE']
+
+        self.settings.load_file(p)
+
+        return os.path.exists(p)
+
+    def get_argument_parser(self):
+        """Returns an argument parser for the command-line interface."""
+
+        parser = argparse.ArgumentParser()
+
+        settings_group = parser.add_argument_group('Settings')
+        settings_group.add_argument('--settings', dest='settings_file',
+            metavar='FILENAME', help='Path to settings file.')
+
+        logging_group = parser.add_argument_group('Logging')
+        logging_group.add_argument('-v', '--verbose', dest='verbose',
+            action='store_true', default=False,
+            help='Print verbose output.')
+        logging_group.add_argument('-l', '--log-file', dest='logfile',
+            metavar='FILENAME', type=argparse.FileType('ab'),
+            help='Filename to write log data to.')
+        logging_group.add_argument('--log-interval', dest='log_interval',
+            action='store_true', default=False,
+            help='Prefix log line with interval from last message rather '
+                'than relative time. Note that this is NOT execution time '
+                'if there are parallel operations.')
+
+        subparser = parser.add_subparsers(dest='action')
+
+        # Register argument action providers with us.
+        for cls in HANDLERS:
+            cls.populate_argparse(subparser)
+
+        return parser
new file mode 100644
--- /dev/null
+++ b/python/mach/mach/terminal.py
@@ -0,0 +1,75 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""This file contains code for interacting with terminals.
+
+All the terminal interaction code is consolidated so the complexity can be in
+one place, away from code that is commonly looked at.
+"""
+
+from __future__ import print_function, unicode_literals
+
+import logging
+import sys
+
+
+class LoggingHandler(logging.Handler):
+    """Custom logging handler that works with terminal window dressing.
+
+    This is alternative terminal logging handler which contains smarts for
+    emitting terminal control characters properly. Currently, it has generic
+    support for "footer" elements at the bottom of the screen. Functionality
+    can be added when needed.
+    """
+    def __init__(self):
+        logging.Handler.__init__(self)
+
+        self.fh = sys.stdout
+        self.footer = None
+
+    def flush(self):
+        self.acquire()
+
+        try:
+            self.fh.flush()
+        finally:
+            self.release()
+
+    def emit(self, record):
+        msg = self.format(record)
+
+        if self.footer:
+            self.footer.clear()
+
+        self.fh.write(msg)
+        self.fh.write('\n')
+
+        if self.footer:
+            self.footer.draw()
+
+        # If we don't flush, the footer may not get drawn.
+        self.flush()
+
+
+class TerminalFooter(object):
+    """Represents something drawn on the bottom of a terminal."""
+    def __init__(self, terminal):
+        self.t = terminal
+        self.fh = sys.stdout
+
+    def _clear_lines(self, n):
+        for i in xrange(n):
+            self.fh.write(self.t.move_x(0))
+            self.fh.write(self.t.clear_eol())
+            self.fh.write(self.t.move_up())
+
+        self.fh.write(self.t.move_down())
+        self.fh.write(self.t.move_x(0))
+
+    def clear(self):
+        raise Exception('clear() must be implemented.')
+
+    def draw(self):
+        raise Exception('draw() must be implemented.')
+
new file mode 100644
--- /dev/null
+++ b/python/mach/setup.py
@@ -0,0 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from setuptools import setup
+
+VERSION = '0.1'
+
+setup(
+    name='mach',
+    description='CLI frontend to mozilla-central.',
+    license='MPL 2.0',
+    packages=['mach'],
+    version=VERSION
+)
+