Merge mozilla-central to mozilla-inbound DONTBUILD since all NPOTB
authorEd Morley <emorley@mozilla.com>
Wed, 05 Dec 2012 23:43:42 +0000
changeset 115138 4cc00705166e693c02fffca115ede5d981ac936f
parent 115137 57473f3eececaa2c2c2af6b9488f0179e3162c0f (current diff)
parent 115077 112c885bd01f504678222b0810df95387e0ab990 (diff)
child 115139 1f3975a90f400b19a6460d270468707066f9e855
push id23973
push useremorley@mozilla.com
push dateThu, 06 Dec 2012 10:04:18 +0000
treeherdermozilla-central@ddda5400c826 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
milestone20.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Merge mozilla-central to mozilla-inbound DONTBUILD since all NPOTB
--- a/python/mach/mach/main.py
+++ b/python/mach/mach/main.py
@@ -309,16 +309,28 @@ To see more help for a specific command,
         except KeyboardInterrupt as ki:
             raise ki
         except Exception as e:
             exc_type, exc_value, exc_tb = sys.exc_info()
 
             # The first frame is us and is never used.
             stack = traceback.extract_tb(exc_tb)[1:]
 
+            # If we have nothing on the stack, the exception was raised as part
+            # of calling the @Command method itself. This likely means a
+            # mismatch between @CommandArgument and arguments to the method.
+            # e.g. there exists a @CommandArgument without the corresponding
+            # argument on the method. We handle that here until the module
+            # loader grows the ability to validate better.
+            if not len(stack):
+                print(COMMAND_ERROR)
+                self._print_exception(sys.stdout, exc_type, exc_value,
+                    traceback.extract_tb(exc_tb))
+                return 1
+
             # Split the frames into those from the module containing the
             # command and everything else.
             command_frames = []
             other_frames = []
 
             initial_file = stack[0][0]
 
             for frame in stack:
--- a/python/mach/mach/mixin/process.py
+++ b/python/mach/mach/mixin/process.py
@@ -38,17 +38,17 @@ if os.environ.get('MSYSTEM', None) == 'M
 
 
 class ProcessExecutionMixin(LoggingMixin):
     """Mix-in that provides process execution functionality."""
 
     def run_process(self, args=None, cwd=None, append_env=None,
         explicit_env=None, log_name=None, log_level=logging.INFO,
         line_handler=None, require_unix_environment=False,
-        ensure_exit_code=0, ignore_children=False):
+        ensure_exit_code=0, ignore_children=False, pass_thru=False):
         """Runs a single process to completion.
 
         Takes a list of arguments to run where the first item is the
         executable. Runs the command in the specified directory and
         with optional environment variables.
 
         append_env -- Dict of environment variables to append to the current
             set of environment variables.
@@ -60,16 +60,23 @@ class ProcessExecutionMixin(LoggingMixin
         execute the command via an appropriate UNIX-like shell.
 
         ignore_children is proxied to mozprocess's ignore_children.
 
         ensure_exit_code is used to ensure the exit code of a process matches
         what is expected. If it is an integer, we raise an Exception if the
         exit code does not match this value. If it is True, we ensure the exit
         code is 0. If it is False, we don't perform any exit code validation.
+
+        pass_thru is a special execution mode where the child process inherits
+        this process's standard file handles (stdin, stdout, stderr) as well as
+        additional file descriptors. It should be used for interactive processes
+        where buffering from mozprocess could be an issue. pass_thru does not
+        use mozprocess. Therefore, arguments like log_name, line_handler,
+        and ignore_children have no effect.
         """
         args = self._normalize_command(args, require_unix_environment)
 
         self.log(logging.INFO, 'new_process', {'args': args}, ' '.join(args))
 
         def handleLine(line):
             # Converts str to unicode on Python 2 and bytes to str on Python 3.
             if isinstance(line, bytes):
@@ -89,22 +96,25 @@ class ProcessExecutionMixin(LoggingMixin
         else:
             use_env.update(os.environ)
 
             if append_env:
                 use_env.update(append_env)
 
         self.log(logging.DEBUG, 'process', {'env': use_env}, 'Environment: {env}')
 
-        p = ProcessHandlerMixin(args, cwd=cwd, env=use_env,
-            processOutputLine=[handleLine], universal_newlines=True,
-            ignore_children=ignore_children)
-        p.run()
-        p.processOutput()
-        status = p.wait()
+        if pass_thru:
+            status = subprocess.call(args, cwd=cwd, env=use_env)
+        else:
+            p = ProcessHandlerMixin(args, cwd=cwd, env=use_env,
+                processOutputLine=[handleLine], universal_newlines=True,
+                ignore_children=ignore_children)
+            p.run()
+            p.processOutput()
+            status = p.wait()
 
         if ensure_exit_code is False:
             return status
 
         if ensure_exit_code is True:
             ensure_exit_code = 0
 
         if status != ensure_exit_code:
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -1,25 +1,29 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import unicode_literals
+from __future__ import print_function, unicode_literals
 
 import logging
 import os
 import subprocess
 import sys
 import which
 
 from mach.mixin.logging import LoggingMixin
 from mach.mixin.process import ProcessExecutionMixin
 
 from .config import BuildConfig
-from .mozconfig import MozconfigLoader
+from .mozconfig import (
+    MozconfigFindException,
+    MozconfigLoadException,
+    MozconfigLoader,
+)
 
 
 class MozbuildObject(ProcessExecutionMixin):
     """Base class providing basic functionality useful to many modules.
 
     Modules in this package typically require common functionality such as
     accessing the current config, getting the location of the source directory,
     running processes, etc. This classes provides that functionality. Other
@@ -117,17 +121,18 @@ class MozbuildObject(ProcessExecutionMix
 
     def _get_objdir_path(self, path):
         """Convert a relative path in the object directory to a full path."""
         return os.path.join(self.topobjdir, path)
 
     def _run_make(self, directory=None, filename=None, target=None, log=True,
             srcdir=False, allow_parallel=True, line_handler=None,
             append_env=None, explicit_env=None, ignore_errors=False,
-            ensure_exit_code=0, silent=True, print_directory=True):
+            ensure_exit_code=0, silent=True, print_directory=True,
+            pass_thru=False):
         """Invoke make.
 
         directory -- Relative directory to look for Makefile in.
         filename -- Explicit makefile to run.
         target -- Makefile target(s) to make. Can be a string or iterable of
             strings.
         srcdir -- If True, invoke make from the source directory tree.
             Otherwise, make will be invoked from the object directory.
@@ -174,16 +179,17 @@ class MozbuildObject(ProcessExecutionMix
         params = {
             'args': args,
             'line_handler': line_handler,
             'append_env': append_env,
             'explicit_env': explicit_env,
             'log_level': logging.INFO,
             'require_unix_environment': True,
             'ensure_exit_code': ensure_exit_code,
+            'pass_thru': pass_thru,
 
             # Make manages its children, so mozprocess doesn't need to bother.
             # Having mozprocess manage children can also have side-effects when
             # building on Windows. See bug 796840.
             'ignore_children': True,
         }
 
         if log:
@@ -237,8 +243,32 @@ class MachCommandBase(MozbuildObject):
 
     This provides a level of indirection so MozbuildObject can be refactored
     without having to change everything that inherits from it.
     """
 
     def __init__(self, context):
         MozbuildObject.__init__(self, context.topdir, context.settings,
             context.log_manager)
+
+        # Incur mozconfig processing so we have unified error handling for
+        # errors. Otherwise, the exceptions could bubble back to mach's error
+        # handler.
+        try:
+            self.mozconfig
+
+        except MozconfigFindException as e:
+            print(e.message)
+            sys.exit(1)
+
+        except MozconfigLoadException as e:
+            print('Error loading mozconfig: ' + e.path)
+            print('')
+            print(e.message)
+            if e.output:
+                print('')
+                print('mozconfig output:')
+                print('')
+                for line in e.output:
+                    print(line)
+
+            sys.exit(1)
+
--- a/python/mozbuild/mozbuild/mozconfig.py
+++ b/python/mozbuild/mozbuild/mozconfig.py
@@ -21,29 +21,36 @@ variable, use MOZCONFIG instead.
 '''.strip()
 
 MOZCONFIG_LEGACY_PATH = '''
 You currently have a mozconfig at %s. This implicit location is no longer
 supported. Please move it to %s/.mozconfig or set an explicit path
 via the $MOZCONFIG environment variable.
 '''.strip()
 
+MOZCONFIG_BAD_EXIT_CODE = '''
+Evaluation of your mozconfig exited with an error. This could be triggered
+by a command inside your mozconfig failing. Please change your mozconfig
+to not error and/or to catch errors in executed commands.
+'''.strip()
+
 
 class MozconfigFindException(Exception):
     """Raised when a mozconfig location is not defined properly."""
 
 
 class MozconfigLoadException(Exception):
     """Raised when a mozconfig could not be loaded properly.
 
     This typically indicates a malformed or misbehaving mozconfig file.
     """
 
-    def __init__(self, path, message):
+    def __init__(self, path, message, output=None):
         self.path = path
+        self.output = output
         Exception.__init__(self, message)
 
 
 class MozconfigLoader(ProcessExecutionMixin):
     """Handles loading and parsing of mozconfig files."""
 
     RE_MAKE_VARIABLE = re.compile('''
         ^\s*                    # Leading whitespace
@@ -168,18 +175,32 @@ class MozconfigLoader(ProcessExecutionMi
         result['make_extra'] = []
         result['make_flags'] = []
 
         env = dict(os.environ)
 
         args = self._normalize_command([self._loader_script, self.topsrcdir,
             path], True)
 
-        output = subprocess.check_output(args, stderr=subprocess.PIPE,
-            cwd=self.topsrcdir, env=env)
+        try:
+            # We need to capture stderr because that's where the shell sends
+            # errors if execution fails.
+            output = subprocess.check_output(args, stderr=subprocess.STDOUT,
+                cwd=self.topsrcdir, env=env)
+        except subprocess.CalledProcessError as e:
+            lines = e.output.splitlines()
+
+            # Output before actual execution shouldn't be relevant.
+            try:
+                index = lines.index('------END_BEFORE_SOURCE')
+                lines = lines[index + 1:]
+            except ValueError:
+                pass
+
+            raise MozconfigLoadException(path, MOZCONFIG_BAD_EXIT_CODE, lines)
 
         parsed = self._parse_loader_output(output)
 
         all_variables = set(parsed['vars_before'].keys())
         all_variables |= set(parsed['vars_after'].keys())
 
         changed = {
             'added': {},
@@ -240,17 +261,17 @@ class MozconfigLoader(ProcessExecutionMi
         ac_options = []
         before_source = {}
         after_source = {}
 
         current = None
         current_type = None
         in_variable = None
 
-        for line in output.split('\n'):
+        for line in output.splitlines():
             if not len(line):
                 continue
 
             if line.startswith('------BEGIN_'):
                 assert current_type is None
                 assert current is None
                 assert not in_variable
                 current_type = line[len('------BEGIN_'):]
--- a/python/mozbuild/mozbuild/test/test_mozconfig.py
+++ b/python/mozbuild/mozbuild/test/test_mozconfig.py
@@ -12,16 +12,17 @@ from shutil import rmtree
 from tempfile import (
     gettempdir,
     mkdtemp,
     NamedTemporaryFile,
 )
 
 from mozbuild.mozconfig import (
     MozconfigFindException,
+    MozconfigLoadException,
     MozconfigLoader,
 )
 
 
 class TestMozconfigLoader(unittest.TestCase):
     def setUp(self):
         self._old_env = dict(os.environ)
         self._temp_dirs = set()
@@ -291,8 +292,23 @@ class TestMozconfigLoader(unittest.TestC
             mozconfig.write('EMPTY=\n')
             mozconfig.flush()
 
             result = self.get_loader().read_mozconfig(mozconfig.name)
 
             self.assertIn('EMPTY', result['env']['added'])
             self.assertEqual(result['env']['added']['EMPTY'], '')
 
+    def test_read_load_exception(self):
+        """Ensure non-0 exit codes in mozconfigs are handled properly."""
+        with NamedTemporaryFile(mode='w') as mozconfig:
+            mozconfig.write('echo "hello world"\n')
+            mozconfig.write('exit 1\n')
+            mozconfig.flush()
+
+            with self.assertRaises(MozconfigLoadException) as e:
+                self.get_loader().read_mozconfig(mozconfig.name)
+
+            self.assertTrue(e.exception.message.startswith(
+                'Evaluation of your mozconfig exited with an error'))
+            self.assertEquals(e.exception.path, mozconfig.name)
+            self.assertEquals(e.exception.output, ['hello world'])
+
--- a/testing/mochitest/mach_commands.py
+++ b/testing/mochitest/mach_commands.py
@@ -18,16 +18,18 @@ from mach.decorators import (
     CommandProvider,
     Command,
 )
 
 
 generic_help = 'Test to run. Can be specified as a single file, a ' +\
 'directory, or omitted. If omitted, the entire test suite is executed.'
 
+debugger_help = 'Debugger binary to run test in. Program name or path.'
+
 
 class MochitestRunner(MozbuildObject):
     """Easily run mochitests.
 
     This currently contains just the basics for running mochitests. We may want
     to hook up result parsing, etc.
     """
     def run_plain_suite(self):
@@ -45,25 +47,28 @@ class MochitestRunner(MozbuildObject):
         # TODO hook up Python harness runner.
         self._run_make(directory='.', target='mochitest-browser-chrome')
 
     def run_all(self):
         self.run_plain_suite()
         self.run_chrome_suite()
         self.run_browser_chrome_suite()
 
-    def run_mochitest_test(self, test_file=None, suite=None):
+    def run_mochitest_test(self, suite=None, test_file=None, debugger=None):
         """Runs a mochitest.
 
         test_file is a path to a test file. It can be a relative path from the
         top source directory, an absolute filename, or a directory containing
         test files.
 
         suite is the type of mochitest to run. It can be one of ('plain',
         'chrome', 'browser').
+
+        debugger is a program name or path to a binary (presumably a debugger)
+        to run the test in. e.g. 'gdb'
         """
 
         # TODO hook up harness via native Python
         target = None
         if suite == 'plain':
             target = 'mochitest-plain'
         elif suite == 'chrome':
             target = 'mochitest-chrome'
@@ -77,43 +82,58 @@ class MochitestRunner(MozbuildObject):
         if test_file:
             path = parse_test_path(test_file, self.topsrcdir)['normalized']
             if not os.path.exists(path):
                 raise Exception('No manifest file was found at %s.' % path)
             env = {'TEST_PATH': path}
         else:
             env = {}
 
+        pass_thru = False
+
+        if debugger:
+            env[b'EXTRA_TEST_ARGS'] = '--debugger=%s' % debugger
+            pass_thru = True
+
         return self._run_make(directory='.', target=target, append_env=env,
-            ensure_exit_code=False)
+            ensure_exit_code=False, pass_thru=pass_thru)
 
 
 @CommandProvider
 class MachCommands(MachCommandBase):
     @Command('mochitest-plain', help='Run a plain mochitest.')
+    @CommandArgument('--debugger', '-d', metavar='DEBUGGER',
+        help=debugger_help)
     @CommandArgument('test_file', default=None, nargs='?', metavar='TEST',
         help=generic_help)
-    def run_mochitest_plain(self, test_file):
-        return self.run_mochitest(test_file, 'plain')
+    def run_mochitest_plain(self, test_file, debugger=None):
+        return self.run_mochitest(test_file, 'plain', debugger=debugger)
 
     @Command('mochitest-chrome', help='Run a chrome mochitest.')
+    @CommandArgument('--debugger', '-d', metavar='DEBUGGER',
+        help=debugger_help)
     @CommandArgument('test_file', default=None, nargs='?', metavar='TEST',
         help=generic_help)
-    def run_mochitest_chrome(self, test_file):
-        return self.run_mochitest(test_file, 'chrome')
+    def run_mochitest_chrome(self, test_file, debugger=None):
+        return self.run_mochitest(test_file, 'chrome', debugger=debugger)
 
     @Command('mochitest-browser', help='Run a mochitest with browser chrome.')
+    @CommandArgument('--debugger', '-d', metavar='DEBUGGER',
+        help=debugger_help)
     @CommandArgument('test_file', default=None, nargs='?', metavar='TEST',
         help=generic_help)
-    def run_mochitest_browser(self, test_file):
-        return self.run_mochitest(test_file, 'browser')
+    def run_mochitest_browser(self, test_file, debugger=None):
+        return self.run_mochitest(test_file, 'browser', debugger=debugger)
 
     @Command('mochitest-a11y', help='Run an a11y mochitest.')
+    @CommandArgument('--debugger', '-d', metavar='DEBUGGER',
+        help=debugger_help)
     @CommandArgument('test_file', default=None, nargs='?', metavar='TEST',
         help=generic_help)
-    def run_mochitest_a11y(self, test_file):
-        return self.run_mochitest(test_file, 'a11y')
+    def run_mochitest_a11y(self, test_file, debugger=None):
+        return self.run_mochitest(test_file, 'a11y', debugger=debugger)
 
-    def run_mochitest(self, test_file, flavor):
+    def run_mochitest(self, test_file, flavor, debugger=None):
         self._ensure_state_subdir_exists('.')
 
         mochitest = self._spawn(MochitestRunner)
-        return mochitest.run_mochitest_test(test_file, flavor)
+        return mochitest.run_mochitest_test(test_file=test_file, suite=flavor,
+            debugger=debugger)
--- a/testing/xpcshell/mach_commands.py
+++ b/testing/xpcshell/mach_commands.py
@@ -36,17 +36,17 @@ class InvalidTestPathError(Exception):
 
 
 class XPCShellRunner(MozbuildObject):
     """Run xpcshell tests."""
     def run_suite(self, **kwargs):
         manifest = os.path.join(self.topobjdir, '_tests', 'xpcshell',
             'xpcshell.ini')
 
-        self._run_xpcshell_harness(manifest=manifest, **kwargs)
+        return self._run_xpcshell_harness(manifest=manifest, **kwargs)
 
     def run_test(self, test_file, debug=False, interactive=False,
         keep_going=False, shuffle=False):
         """Runs an individual xpcshell test."""
 
         if test_file == 'all':
             self.run_suite(debug=debug, interactive=interactive,
                 keep_going=keep_going, shuffle=shuffle)
@@ -78,17 +78,17 @@ class XPCShellRunner(MozbuildObject):
             'keep_going': keep_going,
             'shuffle': shuffle,
             'test_dirs': [test_dir],
         }
 
         if os.path.isfile(test_file):
             args['test_path'] = os.path.basename(test_file)
 
-        self._run_xpcshell_harness(**args)
+        return self._run_xpcshell_harness(**args)
 
     def _run_xpcshell_harness(self, test_dirs=None, manifest=None,
         test_path=None, debug=False, shuffle=False, interactive=False,
         keep_going=False):
 
         # Obtain a reference to the xpcshell test runner.
         import runxpcshelltests
 
@@ -135,21 +135,22 @@ class XPCShellRunner(MozbuildObject):
             if isinstance(v, unicode_type):
                 v = v.encode('utf-8')
 
             if isinstance(k, unicode_type):
                 k = k.encode('utf-8')
 
             filtered_args[k] = v
 
-        # TODO do something with result.
-        xpcshell.runTests(**filtered_args)
+        result = xpcshell.runTests(**filtered_args)
 
         self.log_manager.disable_unstructured()
 
+        return int(not result)
+
 
 @CommandProvider
 class MachCommands(MachCommandBase):
     @Command('xpcshell-test', help='Run an xpcshell test.')
     @CommandArgument('test_file', default='all', nargs='?', metavar='TEST',
         help='Test to run. Can be specified as a single JS file, a directory, '
              'or omitted. If omitted, the entire test suite is executed.')
     @CommandArgument('--debug', '-d', action='store_true',