author | Gregory Szorc <gps@mozilla.com> |
Wed, 10 Oct 2012 11:08:09 -0700 | |
changeset 117728 | dda561124c61e11eee7fab7f286661e489264cb4 |
parent 117727 | e6202ccc349b6533deacd1865e0a31955dec1b65 |
child 117729 | 66d59a4d5a1b76ff43dd4fe6c474e05574a60578 |
push id | 1997 |
push user | akeybl@mozilla.com |
push date | Mon, 07 Jan 2013 21:25:26 +0000 |
treeherder | mozilla-beta@4baf45cdcf21 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | jhammel |
bugs | 799648 |
milestone | 19.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/python/mach/mach/main.py +++ b/python/mach/mach/main.py @@ -287,16 +287,18 @@ To see more help for a specific command, if args.logfile: self.log_manager.add_json_handler(args.logfile) # Up the logging level if requested. log_level = logging.INFO if args.verbose: log_level = logging.DEBUG + self.log_manager.register_structured_logger(logging.getLogger('mach')) + # Always enable terminal logging. The log manager figures out if we are # actually in a TTY or are a pipe and does the right thing. self.log_manager.add_terminal_logging(level=log_level, write_interval=args.log_interval) self.load_settings(args) conf = BuildConfig(self.settings)
new file mode 100644 --- /dev/null +++ b/python/mach/mach/mixin/logging.py @@ -0,0 +1,55 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import logging + + +class LoggingMixin(object): + """Provides functionality to control logging.""" + + def populate_logger(self, name=None): + """Ensure this class instance has a logger associated with it. + + Users of this mixin that call log() will need to ensure self._logger is + a logging.Logger instance before they call log(). This function ensures + self._logger is defined by populating it if it isn't. + """ + if hasattr(self, '_logger'): + return + + if name is None: + name = '.'.join([self.__module__, self.__class__.__name__]) + + self._logger = logging.getLogger(name) + + def log(self, level, action, params, format_str): + """Log a structured log event. + + A structured log event consists of a logging level, a string action, a + dictionary of attributes, and a formatting string. + + The logging level is one of the logging.* constants, such as + logging.INFO. + + The action string is essentially the enumeration of the event. Each + different type of logged event should have a different action. + + The params dict is the metadata constituting the logged event. + + The formatting string is used to convert the structured message back to + human-readable format. Conversion back to human-readable form is + performed by calling format() on this string, feeding into it the dict + of attributes constituting the event. + + Example Usage + ------------- + + self.log(logging.DEBUG, 'login', {'username': 'johndoe'}, + 'User login: {username}') + """ + self._logger.log(level, format_str, + extra={'action': action, 'params': params}) +
new file mode 100644 --- /dev/null +++ b/python/mach/mach/mixin/process.py @@ -0,0 +1,125 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This module provides mixins to perform process execution. + +from __future__ import absolute_import, unicode_literals + +import logging +import os +import subprocess +import sys + +from mozprocess.processhandler import ProcessHandlerMixin + +from .logging import LoggingMixin + + +# Perform detection of operating system environment. This is used by command +# execution. We only do this once to save redundancy. Yes, this can fail module +# loading. That is arguably OK. +if 'SHELL' in os.environ: + _current_shell = os.environ['SHELL'] +elif 'MOZILLABUILD' in os.environ: + _current_shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe' +elif 'COMSPEC' in os.environ: + _current_shell = os.environ['COMSPEC'] +else: + raise Exception('Could not detect environment shell!') + +_in_msys = False + +if os.environ.get('MSYSTEM', None) == 'MINGW32': + _in_msys = True + + if not _current_shell.lower().endswith('.exe'): + _current_shell += '.exe' + + +class ProcessExecutionMixin(LoggingMixin): + """Mix-in that provides process execution functionality.""" + + def run_process(self, args=None, cwd=None, append_env=None, + explicit_env=None, log_name=None, log_level=logging.INFO, + line_handler=None, require_unix_environment=False, + ignore_errors=False, ignore_children=False): + """Runs a single process to completion. + + Takes a list of arguments to run where the first item is the + executable. Runs the command in the specified directory and + with optional environment variables. + + append_env -- Dict of environment variables to append to the current + set of environment variables. + explicit_env -- Dict of environment variables to set for the new + process. Any existing environment variables will be ignored. + + require_unix_environment if True will ensure the command is executed + within a UNIX environment. Basically, if we are on Windows, it will + execute the command via an appropriate UNIX-like shell. + + ignore_children is proxied to mozprocess's ignore_children. + """ + args = self._normalize_command(args, require_unix_environment) + + self.log(logging.INFO, 'new_process', {'args': args}, ' '.join(args)) + + def handleLine(line): + # Converts str to unicode on Python 2 and bytes to str on Python 3. + if isinstance(line, bytes): + line = line.decode(sys.stdout.encoding or 'utf-8', 'replace') + + if line_handler: + line_handler(line) + + if not log_name: + return + + self.log(log_level, log_name, {'line': line.strip()}, '{line}') + + use_env = {} + if explicit_env: + use_env = explicit_env + else: + use_env.update(os.environ) + + if append_env: + use_env.update(append_env) + + self.log(logging.DEBUG, 'process', {'env': use_env}, 'Environment: {env}') + + p = ProcessHandlerMixin(args, cwd=cwd, env=use_env, + processOutputLine=[handleLine], universal_newlines=True, + ignore_children=ignore_children) + p.run() + p.processOutput() + status = p.wait() + + if status != 0 and not ignore_errors: + raise Exception('Process executed with non-0 exit code: %s' % args) + + def _normalize_command(self, args, require_unix_environment): + """Adjust command arguments to run in the necessary environment. + + This exists mainly to facilitate execution of programs requiring a *NIX + shell when running on Windows. The caller specifies whether a shell + environment is required. If it is and we are running on Windows but + aren't running in the UNIX-like msys environment, then we rewrite the + command to execute via a shell. + """ + assert isinstance(args, list) and len(args) + + if not require_unix_environment or not _in_msys: + return args + + # Always munge Windows-style into Unix style for the command. + prog = args[0].replace('\\', '/') + + # PyMake removes the C: prefix. But, things seem to work here + # without it. Not sure what that's about. + + # We run everything through the msys shell. We need to use + # '-c' and pass all the arguments as one argument because that is + # how sh works. + cline = subprocess.list2cmdline([prog] + args[1:])
--- a/python/mozbuild/mozbuild/base.py +++ b/python/mozbuild/mozbuild/base.py @@ -3,49 +3,30 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. from __future__ import unicode_literals import logging import multiprocessing import os import pymake.parser +import subprocess import sys -import subprocess import which -from mozprocess.processhandler import ProcessHandlerMixin from pymake.data import Makefile +from mach.mixin.logging import LoggingMixin +from mach.mixin.process import ProcessExecutionMixin + from mozbuild.config import ConfigProvider from mozbuild.config import PositiveIntegerType -# Perform detection of operating system environment. This is used by command -# execution. We only do this once to save redundancy. Yes, this can fail module -# loading. That is arguably OK. -if 'SHELL' in os.environ: - _current_shell = os.environ['SHELL'] -elif 'MOZILLABUILD' in os.environ: - _current_shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe' -elif 'COMSPEC' in os.environ: - _current_shell = os.environ['COMSPEC'] -else: - raise Exception('Could not detect environment shell!') - -_in_msys = False - -if os.environ.get('MSYSTEM', None) == 'MINGW32': - _in_msys = True - - if not _current_shell.lower().endswith('.exe'): - _current_shell += '.exe' - - -class MozbuildObject(object): +class MozbuildObject(ProcessExecutionMixin): """Base class providing basic functionality useful to many modules. Modules in this package typically require common functionality such as accessing the current config, getting the location of the source directory, running processes, etc. This classes provides that functionality. Other modules can inherit from this class to obtain this functionality easily. """ def __init__(self, topsrcdir, settings, log_manager, topobjdir=None): @@ -53,17 +34,18 @@ class MozbuildObject(object): Instances are bound to a source directory, a ConfigSettings instance, and a LogManager instance. The topobjdir may be passed in as well. If it isn't, it will be calculated from the active mozconfig. """ self.topsrcdir = topsrcdir self.settings = settings self.config = BuildConfig(settings) - self.logger = logging.getLogger(__name__) + + self.populate_logger() self.log_manager = log_manager self._config_guess_output = None self._make = None self._topobjdir = topobjdir @property def topobjdir(self): @@ -82,19 +64,16 @@ class MozbuildObject(object): @property def bindir(self): return os.path.join(self.topobjdir, 'dist', 'bin') @property def statedir(self): return os.path.join(self.topobjdir, '.mozbuild') - def log(self, level, action, params, format_str): - self.logger.log(level, format_str, - extra={'action': action, 'params': params}) def _load_mozconfig(self, path=None): # The mozconfig loader outputs a make file. We parse and load this make # file with pymake and evaluate it in a context similar to client.mk. loader = os.path.join(self.topsrcdir, 'build', 'autoconf', 'mozconfig2client-mk') @@ -276,104 +255,21 @@ class MozbuildObject(object): continue if self._make is None: raise Exception('Could not find suitable make binary!') return self._make def _run_command_in_srcdir(self, **args): - self._run_command(cwd=self.topsrcdir, **args) + self.run_process(cwd=self.topsrcdir, **args) def _run_command_in_objdir(self, **args): - self._run_command(cwd=self.topobjdir, **args) - - def _run_command(self, args=None, cwd=None, append_env=None, - explicit_env=None, log_name=None, log_level=logging.INFO, - line_handler=None, require_unix_environment=False, - ignore_errors=False, ignore_children=False): - """Runs a single command to completion. - - Takes a list of arguments to run where the first item is the - executable. Runs the command in the specified directory and - with optional environment variables. - - append_env -- Dict of environment variables to append to the current - set of environment variables. - explicit_env -- Dict of environment variables to set for the new - process. Any existing environment variables will be ignored. - - require_unix_environment if True will ensure the command is executed - within a UNIX environment. Basically, if we are on Windows, it will - execute the command via an appropriate UNIX-like shell. - - ignore_children is proxied to mozprocess's ignore_children. - """ - args = self._normalize_command(args, require_unix_environment) - - self.log(logging.INFO, 'process', {'args': args}, ' '.join(args)) - - def handleLine(line): - # Converts str to unicode on Python 2 and bytes to str on Python 3. - if isinstance(line, bytes): - line = line.decode(sys.stdout.encoding or 'utf-8', 'replace') - - if line_handler: - line_handler(line) - - if not log_name: - return - - self.log(log_level, log_name, {'line': line.strip()}, '{line}') + self.run_process(cwd=self.topobjdir, **args) - use_env = {} - if explicit_env: - use_env = explicit_env - else: - use_env.update(os.environ) - - if append_env: - use_env.update(append_env) - - self.log(logging.DEBUG, 'process', {'env': use_env}, 'Environment: {env}') - - p = ProcessHandlerMixin(args, cwd=cwd, env=use_env, - processOutputLine=[handleLine], universal_newlines=True, - ignore_children=ignore_children) - p.run() - p.processOutput() - status = p.wait() - - if status != 0 and not ignore_errors: - raise Exception('Process executed with non-0 exit code: %s' % args) - - def _normalize_command(self, args, require_unix_environment): - """Adjust command arguments to run in the necessary environment. - - This exists mainly to facilitate execution of programs requiring a *NIX - shell when running on Windows. The caller specifies whether a shell - environment is required. If it is and we are running on Windows but - aren't running in the UNIX-like msys environment, then we rewrite the - command to execute via a shell. - """ - assert isinstance(args, list) and len(args) - - if not require_unix_environment or not _in_msys: - return args - - # Always munge Windows-style into Unix style for the command. - prog = args[0].replace('\\', '/') - - # PyMake removes the C: prefix. But, things seem to work here - # without it. Not sure what that's about. - - # We run everything through the msys shell. We need to use - # '-c' and pass all the arguments as one argument because that is - # how sh works. - cline = subprocess.list2cmdline([prog] + args[1:]) return [_current_shell, '-c', cline] def _is_windows(self): return os.name in ('nt', 'ce') def _spawn(self, cls): """Create a new MozbuildObject-derived class instance from ourselves.