Bug 799648 - Part 1: Move process execution and logging methods into Mach mixin classes; r=jhammel
authorGregory Szorc <gps@mozilla.com>
Wed, 10 Oct 2012 11:08:09 -0700
changeset 117728 dda561124c61e11eee7fab7f286661e489264cb4
parent 117727 e6202ccc349b6533deacd1865e0a31955dec1b65
child 117729 66d59a4d5a1b76ff43dd4fe6c474e05574a60578
push id1997
push userakeybl@mozilla.com
push dateMon, 07 Jan 2013 21:25:26 +0000
treeherdermozilla-beta@4baf45cdcf21 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjhammel
bugs799648
milestone19.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 799648 - Part 1: Move process execution and logging methods into Mach mixin classes; r=jhammel
python/mach/mach/main.py
python/mach/mach/mixin/__init__.py
python/mach/mach/mixin/logging.py
python/mach/mach/mixin/process.py
python/mozbuild/mozbuild/base.py
--- a/python/mach/mach/main.py
+++ b/python/mach/mach/main.py
@@ -287,16 +287,18 @@ To see more help for a specific command,
         if args.logfile:
             self.log_manager.add_json_handler(args.logfile)
 
         # Up the logging level if requested.
         log_level = logging.INFO
         if args.verbose:
             log_level = logging.DEBUG
 
+        self.log_manager.register_structured_logger(logging.getLogger('mach'))
+
         # Always enable terminal logging. The log manager figures out if we are
         # actually in a TTY or are a pipe and does the right thing.
         self.log_manager.add_terminal_logging(level=log_level,
             write_interval=args.log_interval)
 
         self.load_settings(args)
         conf = BuildConfig(self.settings)
 
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/python/mach/mach/mixin/logging.py
@@ -0,0 +1,55 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import logging
+
+
+class LoggingMixin(object):
+    """Provides functionality to control logging."""
+
+    def populate_logger(self, name=None):
+        """Ensure this class instance has a logger associated with it.
+
+        Users of this mixin that call log() will need to ensure self._logger is
+        a logging.Logger instance before they call log(). This function ensures
+        self._logger is defined by populating it if it isn't.
+        """
+        if hasattr(self, '_logger'):
+            return
+
+        if name is None:
+            name = '.'.join([self.__module__, self.__class__.__name__])
+
+        self._logger = logging.getLogger(name)
+
+    def log(self, level, action, params, format_str):
+        """Log a structured log event.
+
+        A structured log event consists of a logging level, a string action, a
+        dictionary of attributes, and a formatting string.
+
+        The logging level is one of the logging.* constants, such as
+        logging.INFO.
+
+        The action string is essentially the enumeration of the event. Each
+        different type of logged event should have a different action.
+
+        The params dict is the metadata constituting the logged event.
+
+        The formatting string is used to convert the structured message back to
+        human-readable format. Conversion back to human-readable form is
+        performed by calling format() on this string, feeding into it the dict
+        of attributes constituting the event.
+
+        Example Usage
+        -------------
+
+        self.log(logging.DEBUG, 'login', {'username': 'johndoe'},
+            'User login: {username}')
+        """
+        self._logger.log(level, format_str,
+            extra={'action': action, 'params': params})
+
new file mode 100644
--- /dev/null
+++ b/python/mach/mach/mixin/process.py
@@ -0,0 +1,125 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This module provides mixins to perform process execution.
+
+from __future__ import absolute_import, unicode_literals
+
+import logging
+import os
+import subprocess
+import sys
+
+from mozprocess.processhandler import ProcessHandlerMixin
+
+from .logging import LoggingMixin
+
+
+# Perform detection of operating system environment. This is used by command
+# execution. We only do this once to save redundancy. Yes, this can fail module
+# loading. That is arguably OK.
+if 'SHELL' in os.environ:
+    _current_shell = os.environ['SHELL']
+elif 'MOZILLABUILD' in os.environ:
+    _current_shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe'
+elif 'COMSPEC' in os.environ:
+    _current_shell = os.environ['COMSPEC']
+else:
+    raise Exception('Could not detect environment shell!')
+
+_in_msys = False
+
+if os.environ.get('MSYSTEM', None) == 'MINGW32':
+    _in_msys = True
+
+    if not _current_shell.lower().endswith('.exe'):
+        _current_shell += '.exe'
+
+
+class ProcessExecutionMixin(LoggingMixin):
+    """Mix-in that provides process execution functionality."""
+
+    def run_process(self, args=None, cwd=None, append_env=None,
+        explicit_env=None, log_name=None, log_level=logging.INFO,
+        line_handler=None, require_unix_environment=False,
+        ignore_errors=False, ignore_children=False):
+        """Runs a single process to completion.
+
+        Takes a list of arguments to run where the first item is the
+        executable. Runs the command in the specified directory and
+        with optional environment variables.
+
+        append_env -- Dict of environment variables to append to the current
+            set of environment variables.
+        explicit_env -- Dict of environment variables to set for the new
+            process. Any existing environment variables will be ignored.
+
+        require_unix_environment if True will ensure the command is executed
+        within a UNIX environment. Basically, if we are on Windows, it will
+        execute the command via an appropriate UNIX-like shell.
+
+        ignore_children is proxied to mozprocess's ignore_children.
+        """
+        args = self._normalize_command(args, require_unix_environment)
+
+        self.log(logging.INFO, 'new_process', {'args': args}, ' '.join(args))
+
+        def handleLine(line):
+            # Converts str to unicode on Python 2 and bytes to str on Python 3.
+            if isinstance(line, bytes):
+                line = line.decode(sys.stdout.encoding or 'utf-8', 'replace')
+
+            if line_handler:
+                line_handler(line)
+
+            if not log_name:
+                return
+
+            self.log(log_level, log_name, {'line': line.strip()}, '{line}')
+
+        use_env = {}
+        if explicit_env:
+            use_env = explicit_env
+        else:
+            use_env.update(os.environ)
+
+            if append_env:
+                use_env.update(append_env)
+
+        self.log(logging.DEBUG, 'process', {'env': use_env}, 'Environment: {env}')
+
+        p = ProcessHandlerMixin(args, cwd=cwd, env=use_env,
+            processOutputLine=[handleLine], universal_newlines=True,
+            ignore_children=ignore_children)
+        p.run()
+        p.processOutput()
+        status = p.wait()
+
+        if status != 0 and not ignore_errors:
+            raise Exception('Process executed with non-0 exit code: %s' % args)
+
+    def _normalize_command(self, args, require_unix_environment):
+        """Adjust command arguments to run in the necessary environment.
+
+        This exists mainly to facilitate execution of programs requiring a *NIX
+        shell when running on Windows. The caller specifies whether a shell
+        environment is required. If it is and we are running on Windows but
+        aren't running in the UNIX-like msys environment, then we rewrite the
+        command to execute via a shell.
+        """
+        assert isinstance(args, list) and len(args)
+
+        if not require_unix_environment or not _in_msys:
+            return args
+
+        # Always munge Windows-style into Unix style for the command.
+        prog = args[0].replace('\\', '/')
+
+        # PyMake removes the C: prefix. But, things seem to work here
+        # without it. Not sure what that's about.
+
+        # We run everything through the msys shell. We need to use
+        # '-c' and pass all the arguments as one argument because that is
+        # how sh works.
+        cline = subprocess.list2cmdline([prog] + args[1:])
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -3,49 +3,30 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 import logging
 import multiprocessing
 import os
 import pymake.parser
+import subprocess
 import sys
-import subprocess
 import which
 
-from mozprocess.processhandler import ProcessHandlerMixin
 from pymake.data import Makefile
 
+from mach.mixin.logging import LoggingMixin
+from mach.mixin.process import ProcessExecutionMixin
+
 from mozbuild.config import ConfigProvider
 from mozbuild.config import PositiveIntegerType
 
 
-# Perform detection of operating system environment. This is used by command
-# execution. We only do this once to save redundancy. Yes, this can fail module
-# loading. That is arguably OK.
-if 'SHELL' in os.environ:
-    _current_shell = os.environ['SHELL']
-elif 'MOZILLABUILD' in os.environ:
-    _current_shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh.exe'
-elif 'COMSPEC' in os.environ:
-    _current_shell = os.environ['COMSPEC']
-else:
-    raise Exception('Could not detect environment shell!')
-
-_in_msys = False
-
-if os.environ.get('MSYSTEM', None) == 'MINGW32':
-    _in_msys = True
-
-    if not _current_shell.lower().endswith('.exe'):
-        _current_shell += '.exe'
-
-
-class MozbuildObject(object):
+class MozbuildObject(ProcessExecutionMixin):
     """Base class providing basic functionality useful to many modules.
 
     Modules in this package typically require common functionality such as
     accessing the current config, getting the location of the source directory,
     running processes, etc. This classes provides that functionality. Other
     modules can inherit from this class to obtain this functionality easily.
     """
     def __init__(self, topsrcdir, settings, log_manager, topobjdir=None):
@@ -53,17 +34,18 @@ class MozbuildObject(object):
 
         Instances are bound to a source directory, a ConfigSettings instance,
         and a LogManager instance. The topobjdir may be passed in as well. If
         it isn't, it will be calculated from the active mozconfig.
         """
         self.topsrcdir = topsrcdir
         self.settings = settings
         self.config = BuildConfig(settings)
-        self.logger = logging.getLogger(__name__)
+
+        self.populate_logger()
         self.log_manager = log_manager
 
         self._config_guess_output = None
         self._make = None
         self._topobjdir = topobjdir
 
     @property
     def topobjdir(self):
@@ -82,19 +64,16 @@ class MozbuildObject(object):
     @property
     def bindir(self):
         return os.path.join(self.topobjdir, 'dist', 'bin')
 
     @property
     def statedir(self):
         return os.path.join(self.topobjdir, '.mozbuild')
 
-    def log(self, level, action, params, format_str):
-        self.logger.log(level, format_str,
-            extra={'action': action, 'params': params})
 
     def _load_mozconfig(self, path=None):
         # The mozconfig loader outputs a make file. We parse and load this make
         # file with pymake and evaluate it in a context similar to client.mk.
 
         loader = os.path.join(self.topsrcdir, 'build', 'autoconf',
             'mozconfig2client-mk')
 
@@ -276,104 +255,21 @@ class MozbuildObject(object):
                         continue
 
         if self._make is None:
             raise Exception('Could not find suitable make binary!')
 
         return self._make
 
     def _run_command_in_srcdir(self, **args):
-        self._run_command(cwd=self.topsrcdir, **args)
+        self.run_process(cwd=self.topsrcdir, **args)
 
     def _run_command_in_objdir(self, **args):
-        self._run_command(cwd=self.topobjdir, **args)
-
-    def _run_command(self, args=None, cwd=None, append_env=None,
-        explicit_env=None, log_name=None, log_level=logging.INFO,
-        line_handler=None, require_unix_environment=False,
-        ignore_errors=False, ignore_children=False):
-        """Runs a single command to completion.
-
-        Takes a list of arguments to run where the first item is the
-        executable. Runs the command in the specified directory and
-        with optional environment variables.
-
-        append_env -- Dict of environment variables to append to the current
-            set of environment variables.
-        explicit_env -- Dict of environment variables to set for the new
-            process. Any existing environment variables will be ignored.
-
-        require_unix_environment if True will ensure the command is executed
-        within a UNIX environment. Basically, if we are on Windows, it will
-        execute the command via an appropriate UNIX-like shell.
-
-        ignore_children is proxied to mozprocess's ignore_children.
-        """
-        args = self._normalize_command(args, require_unix_environment)
-
-        self.log(logging.INFO, 'process', {'args': args}, ' '.join(args))
-
-        def handleLine(line):
-            # Converts str to unicode on Python 2 and bytes to str on Python 3.
-            if isinstance(line, bytes):
-                line = line.decode(sys.stdout.encoding or 'utf-8', 'replace')
-
-            if line_handler:
-                line_handler(line)
-
-            if not log_name:
-                return
-
-            self.log(log_level, log_name, {'line': line.strip()}, '{line}')
+        self.run_process(cwd=self.topobjdir, **args)
 
-        use_env = {}
-        if explicit_env:
-            use_env = explicit_env
-        else:
-            use_env.update(os.environ)
-
-            if append_env:
-                use_env.update(append_env)
-
-        self.log(logging.DEBUG, 'process', {'env': use_env}, 'Environment: {env}')
-
-        p = ProcessHandlerMixin(args, cwd=cwd, env=use_env,
-            processOutputLine=[handleLine], universal_newlines=True,
-            ignore_children=ignore_children)
-        p.run()
-        p.processOutput()
-        status = p.wait()
-
-        if status != 0 and not ignore_errors:
-            raise Exception('Process executed with non-0 exit code: %s' % args)
-
-    def _normalize_command(self, args, require_unix_environment):
-        """Adjust command arguments to run in the necessary environment.
-
-        This exists mainly to facilitate execution of programs requiring a *NIX
-        shell when running on Windows. The caller specifies whether a shell
-        environment is required. If it is and we are running on Windows but
-        aren't running in the UNIX-like msys environment, then we rewrite the
-        command to execute via a shell.
-        """
-        assert isinstance(args, list) and len(args)
-
-        if not require_unix_environment or not _in_msys:
-            return args
-
-        # Always munge Windows-style into Unix style for the command.
-        prog = args[0].replace('\\', '/')
-
-        # PyMake removes the C: prefix. But, things seem to work here
-        # without it. Not sure what that's about.
-
-        # We run everything through the msys shell. We need to use
-        # '-c' and pass all the arguments as one argument because that is
-        # how sh works.
-        cline = subprocess.list2cmdline([prog] + args[1:])
         return [_current_shell, '-c', cline]
 
     def _is_windows(self):
         return os.name in ('nt', 'ce')
 
     def _spawn(self, cls):
         """Create a new MozbuildObject-derived class instance from ourselves.