Merge mozilla-central into mozilla-inbound
authorEhsan Akhgari <ehsan@mozilla.com>
Wed, 26 Sep 2012 13:12:22 -0400
changeset 108157 ff22c54142371c4e34f06d975df7f4494072f448
parent 108156 7228effb2e5b55055f7d8597a86d42857a270beb (current diff)
parent 108108 2359243ee2b1abe3fdef4c9ceebce262912466ef (diff)
child 108158 be63bddab58b180d875089af902f5aacdd7216d9
push id23539
push userryanvm@gmail.com
push dateWed, 26 Sep 2012 22:55:55 +0000
treeherderautoland@ec079fd92224 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
milestone18.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Merge mozilla-central into mozilla-inbound
--- a/.gitignore
+++ b/.gitignore
@@ -14,16 +14,17 @@ ID
 
 # User files that may appear at the root
 /.mozconfig*
 /mozconfig
 /configure
 /config.cache
 /config.log
 /.clang_complete
+/mach.ini
 
 # Empty marker file that's generated when we check out NSS
 security/manager/.nss.checkout
 
 # Build directories
 /obj*/
 
 # Build directories for js shell
--- a/.hgignore
+++ b/.hgignore
@@ -13,16 +13,17 @@
 
 # User files that may appear at the root
 ^\.mozconfig
 ^mozconfig*
 ^configure$
 ^config\.cache$
 ^config\.log$
 ^\.clang_complete
+^mach.ini$
 
 # Empty marker file that's generated when we check out NSS
 ^security/manager/\.nss\.checkout$
 
 # Build directories
 ^obj
 
 # Build directories for js shell
new file mode 100755
--- /dev/null
+++ b/mach
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import os
+import platform
+import sys
+
+# Ensure we are running Python 2.7+. We put this check here so we generate a
+# user-friendly error message rather than a cryptic stack trace on module
+# import.
+if sys.version_info[0] == 2 and sys.version_info[1] < 7:
+    print('Python 2.7 or above is required to run mach.')
+    print('You are running', platform.python_version())
+    sys.exit(1)
+
+# TODO Bug 794506 Integrate with the in-tree virtualenv configuration.
+SEARCH_PATHS = [
+    'python/mach',
+    'python/mozbuild',
+    'build',
+    'build/pymake',
+    'python/blessings',
+    'python/psutil',
+    'python/which',
+    'other-licenses/ply',
+    'xpcom/idl-parser',
+    'testing/xpcshell',
+    'testing/mozbase/mozprocess',
+    'testing/mozbase/mozinfo',
+]
+
+our_dir = os.path.dirname(os.path.abspath(__file__))
+
+try:
+    import mach.main
+except ImportError:
+    SEARCH_PATHS.reverse()
+    sys.path[0:0] = [os.path.join(our_dir, path) for path in SEARCH_PATHS]
+
+    import mach.main
+
+# All of the code is in a module because EVERYTHING IS A LIBRARY.
+mach = mach.main.Mach(our_dir)
+mach.run(sys.argv[1:])
new file mode 100644
--- /dev/null
+++ b/python/mach/README.rst
@@ -0,0 +1,103 @@
+The mach Driver
+===============
+
+The *mach* driver is the command line interface (CLI) to the source tree.
+
+The *mach* driver is invoked by running the *mach* script or from
+instantiating the *Mach* class from the *mach.main* module.
+
+Implementing mach Commands
+--------------------------
+
+The *mach* driver follows the convention of popular tools like Git,
+Subversion, and Mercurial and provides a common driver for multiple
+sub-commands.
+
+Modules inside *mach* typically contain 1 or more classes which
+inherit from *mach.base.ArgumentProvider*. Modules that inherit from
+this class are hooked up to the *mach* CLI driver. So, to add a new
+sub-command/action to *mach*, one simply needs to create a new class in
+the *mach* package which inherits from *ArgumentProvider*.
+
+Currently, you also need to hook up some plumbing in
+*mach.main.Mach*. In the future, we hope to have automatic detection
+of submodules.
+
+Your command class performs the role of configuring the *mach* frontend
+argument parser as well as providing the methods invoked if a command is
+requested. These methods will take the user-supplied input, do something
+(likely by calling a backend function in a separate module), then format
+output to the terminal.
+
+The plumbing to hook up the arguments to the *mach* driver involves
+light magic. At *mach* invocation time, the driver creates a new
+*argparse* instance. For each registered class that provides commands,
+it calls the *populate_argparse* static method, passing it the parser
+instance.
+
+Your class's *populate_argparse* function should register sub-commands
+with the parser.
+
+For example, say you want to provide the *doitall* command. e.g. *mach
+doitall*. You would create the module *mach.doitall* and this
+module would contain the following class:
+
+    from mach.base import ArgumentProvider
+
+    class DoItAll(ArgumentProvider):
+        def run(self, more=False):
+            print 'I did it!'
+
+        @staticmethod
+        def populate_argparse(parser):
+            # Create the parser to handle the sub-command.
+            p = parser.add_parser('doitall', help='Do it all!')
+
+            p.add_argument('more', action='store_true', default=False,
+                help='Do more!')
+
+            # Tell driver that the handler for this sub-command is the
+            # method *run* on the class *DoItAll*.
+            p.set_defaults(cls=DoItAll, method='run')
+
+The most important line here is the call to *set_defaults*.
+Specifically, the *cls* and *method* parameters, which tell the driver
+which class to instantiate and which method to execute if this command
+is requested.
+
+The specified method will receive all arguments parsed from the command.
+It is important that you use named - not positional - arguments for your
+handler functions or things will blow up. This is because the mach driver
+is using the ``**kwargs`` notation to call the defined method.
+
+In the future, we may provide additional syntactical sugar to make all
+this easier. For example, we may provide decorators on methods to hook
+up commands and handlers.
+
+Minimizing Code in Mach
+-----------------------
+
+Mach is just a frontend. Therefore, code in this package should pertain to
+one of 3 areas:
+
+1. Obtaining user input (parsing arguments, prompting, etc)
+2. Calling into some other Python package
+3. Formatting output
+
+Mach should not contain core logic pertaining to the desired task. If you
+find yourself needing to invent some new functionality, you should implement
+it as a generic package outside of mach and then write a mach shim to call
+into it. There are many advantages to this approach, including reusability
+outside of mach (others may want to write other frontends) and easier testing
+(it is easier to test generic libraries than code that interacts with the
+command line or terminal).
+
+Keeping Frontend Modules Small
+------------------------------
+
+The frontend modules providing mach commands are currently all loaded when
+the mach CLI driver starts. Therefore, there is potential for *import bloat*.
+
+We want the CLI driver to load quickly. So, please delay load external modules
+until they are actually required. In other words, don't use a global
+*import* when you can import from inside a specific command's handler.
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/python/mach/mach/base.py
@@ -0,0 +1,13 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+
+class ArgumentProvider(object):
+    """Base class for classes wishing to provide CLI arguments to mach."""
+
+    @staticmethod
+    def populate_argparse(parser):
+        raise Exception("populate_argparse not implemented.")
new file mode 100644
--- /dev/null
+++ b/python/mach/mach/main.py
@@ -0,0 +1,197 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This module provides functionality for the command-line build tool
+# (mach). It is packaged as a module because everything is a library.
+
+from __future__ import unicode_literals
+
+import argparse
+import logging
+import os
+import sys
+
+from mozbuild.base import BuildConfig
+from mozbuild.config import ConfigSettings
+from mozbuild.logger import LoggingManager
+
+# Import sub-command modules
+# TODO Bug 794509 do this via auto-discovery. Update README once this is
+# done.
+from mach.settings import Settings
+from mach.testing import Testing
+
+# Classes inheriting from ArgumentProvider that provide commands.
+HANDLERS = [
+    Settings,
+    Testing,
+]
+
+# Classes inheriting from ConfigProvider that provide settings.
+# TODO this should come from auto-discovery somehow.
+SETTINGS_PROVIDERS = [
+    BuildConfig,
+]
+
+# Settings for argument parser that don't get proxied to sub-module. i.e. these
+# are things consumed by the driver itself.
+CONSUMED_ARGUMENTS = [
+    'settings_file',
+    'verbose',
+    'logfile',
+    'log_interval',
+    'action',
+    'cls',
+    'method',
+    'func',
+]
+
+class Mach(object):
+    """Contains code for the command-line `mach` interface."""
+
+    USAGE = """%(prog)s subcommand [arguments]
+
+mach provides an interface to performing common developer tasks. You specify
+an action/sub-command and it performs it.
+
+Some common actions are:
+
+    %(prog)s help      Show full help, including the list of all commands.
+    %(prog)s test      Run tests.
+
+To see more help for a specific action, run:
+
+  %(prog)s <command> --help
+"""
+
+    def __init__(self, cwd):
+        assert os.path.isdir(cwd)
+
+        self.cwd = cwd
+        self.log_manager = LoggingManager()
+        self.logger = logging.getLogger(__name__)
+        self.settings = ConfigSettings()
+
+        self.log_manager.register_structured_logger(self.logger)
+
+    def run(self, argv):
+        """Runs mach with arguments provided from the command line."""
+        parser = self.get_argument_parser()
+
+        if not len(argv):
+            # We don't register the usage until here because if it is globally
+            # registered, argparse always prints it. This is not desired when
+            # running with --help.
+            parser.usage = Mach.USAGE
+            parser.print_usage()
+            return 0
+
+        if argv[0] == 'help':
+            parser.print_help()
+            return 0
+
+        args = parser.parse_args(argv)
+
+        # Add JSON logging to a file if requested.
+        if args.logfile:
+            self.log_manager.add_json_handler(args.logfile)
+
+        # Up the logging level if requested.
+        log_level = logging.INFO
+        if args.verbose:
+            log_level = logging.DEBUG
+
+        # Always enable terminal logging. The log manager figures out if we are
+        # actually in a TTY or are a pipe and does the right thing.
+        self.log_manager.add_terminal_logging(level=log_level,
+            write_interval=args.log_interval)
+
+        self.load_settings(args)
+        conf = BuildConfig(self.settings)
+
+        stripped = {k: getattr(args, k) for k in vars(args) if k not in
+            CONSUMED_ARGUMENTS}
+
+        # If the action is associated with a class, instantiate and run it.
+        # All classes must be Base-derived and take the expected argument list.
+        if hasattr(args, 'cls'):
+            cls = getattr(args, 'cls')
+            instance = cls(self.cwd, self.settings, self.log_manager)
+            fn = getattr(instance, getattr(args, 'method'))
+
+        # If the action is associated with a function, call it.
+        elif hasattr(args, 'func'):
+            fn = getattr(args, 'func')
+        else:
+            raise Exception('Dispatch configuration error in module.')
+
+        fn(**stripped)
+
+    def log(self, level, action, params, format_str):
+        """Helper method to record a structured log event."""
+        self.logger.log(level, format_str,
+            extra={'action': action, 'params': params})
+
+    def load_settings(self, args):
+        """Determine which settings files apply and load them.
+
+        Currently, we only support loading settings from a single file.
+        Ideally, we support loading from multiple files. This is supported by
+        the ConfigSettings API. However, that API currently doesn't track where
+        individual values come from, so if we load from multiple sources then
+        save, we effectively do a full copy. We don't want this. Until
+        ConfigSettings does the right thing, we shouldn't expose multi-file
+        loading.
+
+        We look for a settings file in the following locations. The first one
+        found wins:
+
+          1) Command line argument
+          2) Environment variable
+          3) Default path
+        """
+        for provider in SETTINGS_PROVIDERS:
+            provider.register_settings()
+            self.settings.register_provider(provider)
+
+        p = os.path.join(self.cwd, 'mach.ini')
+
+        if args.settings_file:
+            p = args.settings_file
+        elif 'MACH_SETTINGS_FILE' in os.environ:
+            p = os.environ['MACH_SETTINGS_FILE']
+
+        self.settings.load_file(p)
+
+        return os.path.exists(p)
+
+    def get_argument_parser(self):
+        """Returns an argument parser for the command-line interface."""
+
+        parser = argparse.ArgumentParser()
+
+        settings_group = parser.add_argument_group('Settings')
+        settings_group.add_argument('--settings', dest='settings_file',
+            metavar='FILENAME', help='Path to settings file.')
+
+        logging_group = parser.add_argument_group('Logging')
+        logging_group.add_argument('-v', '--verbose', dest='verbose',
+            action='store_true', default=False,
+            help='Print verbose output.')
+        logging_group.add_argument('-l', '--log-file', dest='logfile',
+            metavar='FILENAME', type=argparse.FileType('ab'),
+            help='Filename to write log data to.')
+        logging_group.add_argument('--log-interval', dest='log_interval',
+            action='store_true', default=False,
+            help='Prefix log line with interval from last message rather '
+                'than relative time. Note that this is NOT execution time '
+                'if there are parallel operations.')
+
+        subparser = parser.add_subparsers(dest='action')
+
+        # Register argument action providers with us.
+        for cls in HANDLERS:
+            cls.populate_argparse(subparser)
+
+        return parser
new file mode 100644
--- /dev/null
+++ b/python/mach/mach/settings.py
@@ -0,0 +1,51 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+from textwrap import TextWrapper
+
+from mozbuild.base import MozbuildObject
+from mach.base import ArgumentProvider
+
+class Settings(MozbuildObject, ArgumentProvider):
+    """Interact with settings for mach.
+
+    Currently, we only provide functionality to view what settings are
+    available. In the future, this module will be used to modify settings, help
+    people create configs via a wizard, etc.
+    """
+    def list_settings(self):
+        """List available settings in a concise list."""
+        for section in sorted(self.settings):
+            for option in sorted(self.settings[section]):
+                short, full = self.settings.option_help(section, option)
+                print('%s.%s -- %s' % (section, option, short))
+
+    def create(self):
+        """Create an empty settings file with full documentation."""
+        wrapper = TextWrapper(initial_indent='# ', subsequent_indent='# ')
+
+        for section in sorted(self.settings):
+            print('[%s]' % section)
+            print('')
+
+            for option in sorted(self.settings[section]):
+                short, full = self.settings.option_help(section, option)
+
+                print(wrapper.fill(full))
+                print(';%s =' % option)
+                print('')
+
+    @staticmethod
+    def populate_argparse(parser):
+        lst = parser.add_parser('settings-list',
+            help='Show available config settings.')
+
+        lst.set_defaults(cls=Settings, method='list_settings')
+
+        create = parser.add_parser('settings-create',
+            help='Print a new settings file with usage info.')
+
+        create.set_defaults(cls=Settings, method='create')
new file mode 100644
--- /dev/null
+++ b/python/mach/mach/terminal.py
@@ -0,0 +1,75 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""This file contains code for interacting with terminals.
+
+All the terminal interaction code is consolidated so the complexity can be in
+one place, away from code that is commonly looked at.
+"""
+
+from __future__ import print_function, unicode_literals
+
+import logging
+import sys
+
+
+class LoggingHandler(logging.Handler):
+    """Custom logging handler that works with terminal window dressing.
+
+    This is alternative terminal logging handler which contains smarts for
+    emitting terminal control characters properly. Currently, it has generic
+    support for "footer" elements at the bottom of the screen. Functionality
+    can be added when needed.
+    """
+    def __init__(self):
+        logging.Handler.__init__(self)
+
+        self.fh = sys.stdout
+        self.footer = None
+
+    def flush(self):
+        self.acquire()
+
+        try:
+            self.fh.flush()
+        finally:
+            self.release()
+
+    def emit(self, record):
+        msg = self.format(record)
+
+        if self.footer:
+            self.footer.clear()
+
+        self.fh.write(msg)
+        self.fh.write('\n')
+
+        if self.footer:
+            self.footer.draw()
+
+        # If we don't flush, the footer may not get drawn.
+        self.flush()
+
+
+class TerminalFooter(object):
+    """Represents something drawn on the bottom of a terminal."""
+    def __init__(self, terminal):
+        self.t = terminal
+        self.fh = sys.stdout
+
+    def _clear_lines(self, n):
+        for i in xrange(n):
+            self.fh.write(self.t.move_x(0))
+            self.fh.write(self.t.clear_eol())
+            self.fh.write(self.t.move_up())
+
+        self.fh.write(self.t.move_down())
+        self.fh.write(self.t.move_x(0))
+
+    def clear(self):
+        raise Exception('clear() must be implemented.')
+
+    def draw(self):
+        raise Exception('draw() must be implemented.')
+
new file mode 100644
--- /dev/null
+++ b/python/mach/mach/testing.py
@@ -0,0 +1,78 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+from mach.base import ArgumentProvider
+from mozbuild.base import MozbuildObject
+
+
+generic_help = 'Test to run. Can be specified as a single JS file, a ' +\
+'directory, or omitted. If omitted, the entire test suite is executed.'
+
+
+class Testing(MozbuildObject, ArgumentProvider):
+    """Provides commands for running tests."""
+
+    def run_suite(self, suite):
+        from mozbuild.testing.suite import Suite
+
+        s = self._spawn(Suite)
+        s.run_suite(suite)
+
+    def run_mochitest(self, test_file, flavor):
+        from mozbuild.testing.mochitest import MochitestRunner
+
+        mochitest = self._spawn(MochitestRunner)
+        mochitest.run_mochitest_test(test_file, flavor)
+
+    def run_xpcshell_test(self, **params):
+        from mozbuild.testing.xpcshell import XPCShellRunner
+
+        xpcshell = self._spawn(XPCShellRunner)
+        xpcshell.run_test(**params)
+
+    @staticmethod
+    def populate_argparse(parser):
+        # Whole suites.
+        group = parser.add_parser('test', help="Perform tests.")
+
+        suites = set(['xpcshell', 'mochitest-plain', 'mochitest-chrome',
+            'mochitest-browser', 'all'])
+
+        group.add_argument('suite', default='all', choices=suites, nargs='?',
+            help="Test suite to run.")
+
+        group.set_defaults(cls=Testing, method='run_suite', suite='all')
+
+        mochitest_plain = parser.add_parser('mochitest-plain',
+            help='Run a plain mochitest.')
+        mochitest_plain.add_argument('test_file', default='all', nargs='?',
+            metavar='TEST', help=generic_help)
+        mochitest_plain.set_defaults(cls=Testing, method='run_mochitest',
+            flavor='plain')
+
+        mochitest_chrome = parser.add_parser('mochitest-chrome',
+            help='Run a chrome mochitest.')
+        mochitest_chrome.add_argument('test_file', default='all', nargs='?',
+            metavar='TEST', help=generic_help)
+        mochitest_chrome.set_defaults(cls=Testing, method='run_mochitest',
+            flavor='chrome')
+
+        mochitest_browser = parser.add_parser('mochitest-browser',
+            help='Run a mochitest with browser chrome.')
+        mochitest_browser.add_argument('test_file', default='all', nargs='?',
+            metavar='TEST', help=generic_help)
+        mochitest_browser.set_defaults(cls=Testing, method='run_mochitest',
+            flavor='browser')
+
+        xpcshell = parser.add_parser('xpcshell-test',
+            help="Run an individual xpcshell test.")
+
+        xpcshell.add_argument('test_file', default='all', nargs='?',
+            metavar='TEST', help=generic_help)
+        xpcshell.add_argument('--debug', '-d', action='store_true',
+            help='Run test in debugger.')
+
+        xpcshell.set_defaults(cls=Testing, method='run_xpcshell_test')
new file mode 100644
--- /dev/null
+++ b/python/mach/setup.py
@@ -0,0 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from setuptools import setup
+
+VERSION = '0.1'
+
+setup(
+    name='mach',
+    description='CLI frontend to mozilla-central.',
+    license='MPL 2.0',
+    packages=['mach'],
+    version=VERSION
+)
+
--- a/python/mozbuild/README.rst
+++ b/python/mozbuild/README.rst
@@ -7,16 +7,17 @@ build system.
 
 Modules Overview
 ================
 
 * mozbuild.compilation -- Functionality related to compiling. This
   includes managing compiler warnings.
 * mozbuild.logging -- Defines mozbuild's logging infrastructure.
   mozbuild uses a structured logging backend.
+* mozbuild.testing -- Interfaces for running tests.
 
 Structured Logging
 ==================
 
 One of the features of mozbuild is structured logging. Instead of
 conventional logging where simple strings are logged, the internal
 logging mechanism logs all events with the following pieces of
 information:
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/base.py
@@ -0,0 +1,363 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import logging
+import multiprocessing
+import os
+import pymake.parser
+import shlex
+import subprocess
+import which
+
+from mozprocess.processhandler import ProcessHandlerMixin
+from pymake.data import Makefile
+from tempfile import TemporaryFile
+
+from mozbuild.config import ConfigProvider
+from mozbuild.config import PositiveIntegerType
+
+
+# Perform detection of operating system environment. This is used by command
+# execution. We only do this once to save redundancy. Yes, this can fail module
+# loading. That is arguably OK.
+if 'SHELL' in os.environ:
+    _current_shell = os.environ['SHELL']
+elif 'MOZILLABUILD' in os.environ:
+    _current_shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe'
+elif 'COMSPEC' in os.environ:
+    _current_shell = os.environ['COMSPEC']
+else:
+    raise Exception('Could not detect environment shell!')
+
+_in_msys = False
+
+if os.environ.get('MSYSTEM', None) == 'MINGW32':
+    _in_msys = True
+
+    if not _current_shell.lower().endswith('.exe'):
+        _current_shell += '.exe'
+
+
+class MozbuildObject(object):
+    """Base class providing basic functionality useful to many modules.
+
+    Modules in this package typically require common functionality such as
+    accessing the current config, getting the location of the source directory,
+    running processes, etc. This classes provides that functionality. Other
+    modules can inherit from this class to obtain this functionality easily.
+    """
+    def __init__(self, topsrcdir, settings, log_manager, topobjdir=None):
+        """Create a new Mozbuild object instance.
+
+        Instances are bound to a source directory, a ConfigSettings instance,
+        and a LogManager instance. The topobjdir may be passed in as well. If
+        it isn't, it will be calculated from the active mozconfig.
+        """
+        self.topsrcdir = topsrcdir
+        self.settings = settings
+        self.config = BuildConfig(settings)
+        self.logger = logging.getLogger(__name__)
+        self.log_manager = log_manager
+
+        self._config_guess_output = None
+        self._make = None
+        self._topobjdir = topobjdir
+
+    @property
+    def topobjdir(self):
+        if self._topobjdir is None:
+            self._load_mozconfig()
+
+        if self._topobjdir is None:
+            self._topobjdir = 'obj-%s' % self._config_guess
+
+        return self._topobjdir
+
+    @property
+    def distdir(self):
+        return os.path.join(self.topobjdir, 'dist')
+
+    @property
+    def bindir(self):
+        return os.path.join(self.topobjdir, 'dist', 'bin')
+
+    @property
+    def statedir(self):
+        return os.path.join(self.topobjdir, '.mozbuild')
+
+    def log(self, level, action, params, format_str):
+        self.logger.log(level, format_str,
+            extra={'action': action, 'params': params})
+
+    def _load_mozconfig(self, path=None):
+        # The mozconfig loader outputs a make file. We parse and load this make
+        # file with pymake and evaluate it in a context similar to client.mk.
+
+        loader = os.path.join(self.topsrcdir, 'build', 'autoconf',
+            'mozconfig2client-mk')
+
+        # os.environ from a library function is somewhat evil. But, mozconfig
+        # files are tightly coupled with the environment by definition. In the
+        # future, perhaps we'll have a more sanitized environment for mozconfig
+        # execution.
+        env = dict(os.environ)
+        if path is not None:
+            env['MOZCONFIG'] = path
+
+        env['CONFIG_GUESS'] = self._config_guess
+
+        output = subprocess.check_output([loader, self.topsrcdir],
+            stderr=subprocess.PIPE, cwd=self.topsrcdir, env=env)
+
+        # The output is make syntax. We parse this in a specialized make
+        # context.
+        statements = pymake.parser.parsestring(output, 'mozconfig')
+
+        makefile = Makefile(workdir=self.topsrcdir, env={
+            'TOPSRCDIR': self.topsrcdir,
+            'CONFIG_GUESS': self._config_guess})
+
+        statements.execute(makefile)
+
+        def get_value(name):
+            exp = makefile.variables.get(name)[2]
+
+            return exp.resolvestr(makefile, makefile.variables)
+
+        for name, flavor, source, value in makefile.variables:
+            # We only care about variables that came from the parsed mozconfig.
+            if source != pymake.data.Variables.SOURCE_MAKEFILE:
+                continue
+
+            # Ignore some pymake built-ins.
+            if name in ('.PYMAKE', 'MAKELEVEL', 'MAKEFLAGS'):
+                continue
+
+            if name == 'MOZ_OBJDIR':
+                self._topobjdir = get_value(name)
+
+            # If we want to extract other variables defined by mozconfig, here
+            # is where we'd do it.
+
+    @property
+    def _config_guess(self):
+        if self._config_guess_output is None:
+            p = os.path.join(self.topsrcdir, 'build', 'autoconf',
+                'config.guess')
+            self._config_guess_output = subprocess.check_output([p],
+                cwd=self.topsrcdir).strip()
+
+        return self._config_guess_output
+
+    def _ensure_objdir_exists(self):
+        if os.path.isdir(self.statedir):
+            return
+
+        os.makedirs(self.statedir)
+
+    def _ensure_state_subdir_exists(self, subdir):
+        path = os.path.join(self.statedir, subdir)
+
+        if os.path.isdir(path):
+            return
+
+        os.makedirs(path)
+
+    def _get_state_filename(self, filename, subdir=None):
+        path = self.statedir
+
+        if subdir:
+            path = os.path.join(path, subdir)
+
+        return os.path.join(path, filename)
+
+    def _get_srcdir_path(self, path):
+        """Convert a relative path in the source directory to a full path."""
+        return os.path.join(self.topsrcdir, path)
+
+    def _get_objdir_path(self, path):
+        """Convert a relative path in the object directory to a full path."""
+        return os.path.join(self.topobjdir, path)
+
+    def _run_make(self, directory=None, filename=None, target=None, log=True,
+            srcdir=False, allow_parallel=True, line_handler=None, env=None,
+            ignore_errors=False):
+        """Invoke make.
+
+        directory -- Relative directory to look for Makefile in.
+        filename -- Explicit makefile to run.
+        target -- Makefile target(s) to make. Can be a string or iterable of
+            strings.
+        srcdir -- If True, invoke make from the source directory tree.
+            Otherwise, make will be invoked from the object directory.
+        """
+        self._ensure_objdir_exists()
+
+        args = [self._make_path]
+
+        if directory:
+            args.extend(['-C', directory])
+
+        if filename:
+            args.extend(['-f', filename])
+
+        if allow_parallel:
+            args.append('-j%d' % self.settings.build.threads)
+
+        if ignore_errors:
+            args.append('-k')
+
+        # Silent mode by default.
+        args.append('-s')
+
+        # Print entering/leaving directory messages. Some consumers look at
+        # these to measure progress. Ideally, we'd do everything with pymake
+        # and use hooks in its API. Unfortunately, it doesn't provide that
+        # feature... yet.
+        args.append('-w')
+
+        if isinstance(target, list):
+            args.extend(target)
+        elif target:
+            args.append(target)
+
+        fn = self._run_command_in_objdir
+
+        if srcdir:
+            fn = self._run_command_in_srcdir
+
+        params = {
+            'args': args,
+            'line_handler': line_handler,
+            'explicit_env': env,
+            'log_level': logging.INFO,
+            'require_unix_environment': True,
+            'ignore_errors': ignore_errors,
+        }
+
+        if log:
+            params['log_name'] = 'make'
+
+        fn(**params)
+
+    @property
+    def _make_path(self):
+        if self._make is None:
+            if self._is_windows():
+                self._make = os.path.join(self.topsrcdir, 'build', 'pymake',
+                    'make.py')
+
+            else:
+                for test in ['gmake', 'make']:
+                    try:
+                        self._make = which.which(test)
+                        break
+                    except which.WhichError:
+                        continue
+
+        if self._make is None:
+            raise Exception('Could not find suitable make binary!')
+
+        return self._make
+
+    def _run_command_in_srcdir(self, **args):
+        self._run_command(cwd=self.topsrcdir, **args)
+
+    def _run_command_in_objdir(self, **args):
+        self._run_command(cwd=self.topobjdir, **args)
+
+    def _run_command(self, args=None, cwd=None, append_env=None,
+        explicit_env=None, log_name=None, log_level=logging.INFO,
+        line_handler=None, require_unix_environment=False,
+        ignore_errors=False):
+        """Runs a single command to completion.
+
+        Takes a list of arguments to run where the first item is the
+        executable. Runs the command in the specified directory and
+        with optional environment variables.
+
+        append_env -- Dict of environment variables to append to the current
+            set of environment variables.
+        explicit_env -- Dict of environment variables to set for the new
+            process. Any existing environment variables will be ignored.
+
+        require_unix_environment if True will ensure the command is executed
+        within a UNIX environment. Basically, if we are on Windows, it will
+        execute the command via an appropriate UNIX-like shell.
+        """
+        assert isinstance(args, list) and len(args)
+
+        if require_unix_environment and _in_msys:
+            # Always munge Windows-style into Unix style for the command.
+            prog = args[0].replace('\\', '/')
+
+            # PyMake removes the C: prefix. But, things seem to work here
+            # without it. Not sure what that's about.
+
+            # We run everything through the msys shell. We need to use
+            # '-c' and pass all the arguments as one argument because that is
+            # how sh works.
+            cline = subprocess.list2cmdline([prog] + args[1:])
+            args = [_current_shell, '-c', cline]
+
+        self.log(logging.INFO, 'process', {'args': args}, ' '.join(args))
+
+        def handleLine(line):
+            if line_handler:
+                line_handler(line)
+
+            if not log_name:
+                return
+
+            self.log(log_level, log_name, {'line': line.strip()}, '{line}')
+
+        use_env = {}
+        if explicit_env:
+            use_env = explicit_env
+        else:
+            use_env.update(os.environ)
+
+            if append_env:
+                use_env.update(env)
+
+        p = ProcessHandlerMixin(args, cwd=cwd, env=use_env,
+            processOutputLine=[handleLine], universal_newlines=True)
+        p.run()
+        p.processOutput()
+        status = p.wait()
+
+        if status != 0 and not ignore_errors:
+            raise Exception('Process executed with non-0 exit code: %s' % args)
+
+    def _is_windows(self):
+        return os.name in ('nt', 'ce')
+
+    def _spawn(self, cls):
+        """Create a new MozbuildObject-derived class instance from ourselves.
+
+        This is used as a convenience method to create other
+        MozbuildObject-derived class instances. It can only be used on
+        classes that have the same constructor arguments as us.
+        """
+
+        return cls(self.topsrcdir, self.settings, self.log_manager,
+            topobjdir=self.topobjdir)
+
+
+class BuildConfig(ConfigProvider):
+    """The configuration for mozbuild."""
+
+    def __init__(self, settings):
+        self.settings = settings
+
+    @classmethod
+    def _register_settings(cls):
+        def register(section, option, type_cls, **kwargs):
+            cls.register_setting(section, option, type_cls, domain='mozbuild',
+                **kwargs)
+
+        register('build', 'threads', PositiveIntegerType,
+            default=multiprocessing.cpu_count())
new file mode 100644
index 0000000000000000000000000000000000000000..be7711cb2fcfc927de59407bac54fb613dfab0fe
GIT binary patch
literal 301
zc${6%K~4iP3<b~uwL4a2+r>XXH78)#3v|&QAf54$2v1zu$y9I;?!c)yOC1#nfu*-`
zV*7t@R-c{MquaWjTe}yxaR>M82KVKz?#bby`%`Vcwazv1*q5jjM(d}_{I6wEv~<Tv
z$aOxGLMHsYp$MuGu8}e>q?72J@+<a-6H+deEiHr+?9F-{V=@eqt*Ee%`a-6Kxb3?H
w4a~&pBa|e$eI|@Fg~|m>;7UmMk>3o1D4CzZyHv#-hL|$-9<)WnUTQAg4eLHs%>V!Z
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.po
@@ -0,0 +1,8 @@
+msgid "build.threads.short"
+msgstr "Thread Count"
+
+msgid "build.threads.full"
+msgstr "The number of threads to use when performing CPU intensive tasks. "
+       "This constrols the level of parallelization. The default value is "
+       "the number of cores in your machine."
+
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/test_base.py
@@ -0,0 +1,61 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+import unittest
+
+from tempfile import NamedTemporaryFile
+
+from mozbuild.base import BuildConfig
+from mozbuild.base import MozbuildObject
+from mozbuild.config import ConfigSettings
+from mozbuild.logger import LoggingManager
+
+
+curdir = os.path.dirname(__file__)
+topsrcdir = os.path.normpath(os.path.join(curdir, '..', '..', '..', '..'))
+log_manager = LoggingManager()
+
+
+class TestBuildConfig(unittest.TestCase):
+    def test_basic(self):
+        c = ConfigSettings()
+        c.register_provider(BuildConfig)
+
+        c.build.threads = 6
+
+
+class TestMozbuildObject(unittest.TestCase):
+    def get_base(self):
+        settings = ConfigSettings()
+        settings.register_provider(BuildConfig)
+
+        return MozbuildObject(topsrcdir, settings, log_manager)
+
+    def test_mozconfig_parsing(self):
+        with NamedTemporaryFile(mode='wt') as mozconfig:
+            mozconfig.write('mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/some-objdir')
+            mozconfig.flush()
+
+            os.environ['MOZCONFIG'] = mozconfig.name
+
+            base = self.get_base()
+            base._load_mozconfig()
+
+            self.assertEqual(base.topobjdir, '%s/some-objdir' % topsrcdir)
+
+        del os.environ['MOZCONFIG']
+
+    def test_objdir_config_guess(self):
+        base = self.get_base()
+
+        with NamedTemporaryFile() as mozconfig:
+            os.environ['MOZCONFIG'] = mozconfig.name
+
+            self.assertIsNotNone(base.topobjdir)
+            self.assertEqual(len(base.topobjdir.split()), 1)
+
+        del os.environ['MOZCONFIG']
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/testing/mochitest.py
@@ -0,0 +1,82 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+
+from mozbuild.base import MozbuildObject
+
+
+class MochitestRunner(MozbuildObject):
+    """Easily run mochitests.
+
+    This currently contains just the basics for running mochitests. We may want
+    to hook up result parsing, etc.
+    """
+    def run_plain_suite(self):
+        """Runs all plain mochitests."""
+        # TODO hook up Python harness runner.
+        self._run_make(directory='.', target='mochitest-plain')
+
+    def run_chrome_suite(self):
+        """Runs all chrome mochitests."""
+        # TODO hook up Python harness runner.
+        self._run_make(directory='.', target='mochitest-chrome')
+
+    def run_browser_chrome_suite(self):
+        """Runs browser chrome mochitests."""
+        # TODO hook up Python harness runner.
+        self._run_make(directory='.', target='mochitest-browser-chrome')
+
+    def run_all(self):
+        self.run_plain_suite()
+        self.run_chrome_suite()
+        self.run_browser_chrome_suite()
+
+    def run_mochitest_test(self, test_file=None, suite=None):
+        """Runs a mochitest.
+
+        test_file is a path to a test file. It can be a relative path from the
+        top source directory, an absolute filename, or a directory containing
+        test files.
+
+        suite is the type of mochitest to run. It can be one of ('plain',
+        'chrome', 'browser').
+        """
+        if test_file is None:
+            raise Exception('test_file must be defined.')
+
+        parsed = self._parse_test_path(test_file)
+
+        # TODO hook up harness via native Python
+        target = None
+        if suite == 'plain':
+            target = 'mochitest-plain'
+        elif suite == 'chrome':
+            target = 'mochitest-chrome'
+        elif suite == 'browser':
+            target = 'mochitest-browser-chrome'
+        else:
+            raise Exception('None or unrecognized mochitest suite type.')
+
+        env = {'TEST_PATH': parsed['normalized']}
+
+        self._run_make(directory='.', target=target, env=env)
+
+    def _parse_test_path(self, test_path):
+        is_dir = os.path.isdir(test_path)
+
+        if is_dir and not test_path.endswith(os.path.sep):
+            test_path += os.path.sep
+
+        normalized = test_path
+
+        if test_path.startswith(self.topsrcdir):
+            normalized = test_path[len(self.topsrcdir):]
+
+        return {
+            'normalized': normalized,
+            'is_dir': is_dir,
+        }
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/testing/suite.py
@@ -0,0 +1,49 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+from mozbuild.base import MozbuildObject
+from mozbuild.testing.xpcshell import XPCShellRunner
+from mozbuild.testing.mochitest import MochitestRunner
+
+
+class Suite(MozbuildObject):
+    def run_suite(self, suite):
+        """Run a named test suite.
+
+        Recognized names are:
+
+          all - All test suites
+          mochitest-plain - Plain mochitests
+          mochitest-chrome - mochitests with chrome
+          mochitest-browser - mochitests with browser chrome
+          xpcshell - xpcshell tests
+
+        TODO support for other test suite types.
+        """
+
+        xpcshell = self._spawn(XPCShellRunner)
+        mochitest = self._spawn(MochitestRunner)
+
+        if suite == 'all':
+            xpcshell.run_suite()
+            mochitest.run_plain_suite()
+            mochitest.run_chrome_suite()
+            mochitest.run_browser_chrome_suite()
+            return
+
+        m = {
+            'xpcshell': xpcshell.run_suite,
+            'mochitest-plain': mochitest.run_plain_suite,
+            'mochitest-chrome': mochitest.run_chrome_suite,
+            'mochitest-browser': mochitest.run_browser_chrome_suite,
+        }
+
+        method = m.get(suite, None)
+
+        if method is None:
+            raise Exception('Unknown test suite: %s' % suite)
+
+        return method()
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/testing/xpcshell.py
@@ -0,0 +1,91 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This modules contains code for interacting with xpcshell tests.
+
+from __future__ import unicode_literals
+
+import os.path
+
+from StringIO import StringIO
+
+from mozbuild.base import MozbuildObject
+
+
+class XPCShellRunner(MozbuildObject):
+    """Run xpcshell tests."""
+    def run_suite(self):
+        # TODO hook up to harness runner and support things like shuffle,
+        # proper progress updates, etc.
+        self._run_make(directory='.', target='xpcshell-tests')
+
+    def run_test(self, test_file, debug=False):
+        """Runs an individual xpcshell test."""
+        if test_file == 'all':
+            self.run_suite()
+            return
+
+        # dirname() gets confused if there isn't a trailing slash.
+        if os.path.isdir(test_file) and not test_file.endswith(os.path.sep):
+            test_file += os.path.sep
+
+        relative_dir = test_file
+
+        if test_file.startswith(self.topsrcdir):
+            relative_dir = test_file[len(self.topsrcdir):]
+
+        test_dir = os.path.join(self.topobjdir, '_tests', 'xpcshell',
+                os.path.dirname(relative_dir))
+
+        args = {
+            'debug': debug,
+            'test_dirs': [test_dir],
+        }
+
+        if os.path.isfile(test_file):
+            args['test_path'] = os.path.basename(test_file)
+
+        self._run_xpcshell_harness(**args)
+
+    def _run_xpcshell_harness(self, test_dirs=None, manifest=None,
+        test_path=None, debug=False):
+
+        # Obtain a reference to the xpcshell test runner.
+        import runxpcshelltests
+
+        dummy_log = StringIO()
+        xpcshell = runxpcshelltests.XPCShellTests(log=dummy_log)
+        self.log_manager.enable_unstructured()
+
+        tests_dir = os.path.join(self.topobjdir, '_tests', 'xpcshell')
+        modules_dir = os.path.join(self.topobjdir, '_tests', 'modules')
+
+        args = {
+            'xpcshell': os.path.join(self.bindir, 'xpcshell'),
+            'mozInfo': os.path.join(self.topobjdir, 'mozinfo.json'),
+            'symbolsPath': os.path.join(self.distdir, 'crashreporter-symbols'),
+            'logfiles': False,
+            'testsRootDir': tests_dir,
+            'testingModulesDir': modules_dir,
+            'profileName': 'firefox',
+            'verbose': test_path is not None,
+        }
+
+        if manifest is not None:
+            args['manifest'] = manifest
+        elif test_dirs is not None:
+            if isinstance(test_dirs, list):
+                args['testdirs'] = test_dirs
+            else:
+                args['testdirs'] = [test_dirs]
+        else:
+            raise Exception('One of test_dirs or manifest must be provided.')
+
+        if test_path is not None:
+            args['testPath'] = test_path
+
+        # TODO do something with result.
+        xpcshell.runTests(**args)
+
+        self.log_manager.disable_unstructured()