Bug 1412460 - Move build output processing code out of mach_commands.py; r?build draft
authorGregory Szorc <gps@mozilla.com>
Fri, 27 Oct 2017 10:45:41 -0700
changeset 688040 7126b5ad51b31465a7773c5c1945281060bb0473
parent 688039 6bdd34b3caa370ba103699c9f03b621baf9c1ad6
child 688041 922d8dd8a84aaedde59e38efa553d9efffcbb688
push id86650
push usergszorc@mozilla.com
push dateSat, 28 Oct 2017 01:33:47 +0000
reviewersbuild
bugs1412460
milestone58.0a1
Bug 1412460 - Move build output processing code out of mach_commands.py; r?build In general, we shouldn't have so much business logic in mach_commands.py files. I'd like to move more of the low-level "perform a build" logic out of mach_commands.py to facilitate some future work. That code makes use of this output processing code. So as the first step in moving the build code, we move its dependencies. As part of this, I also cleaned up some random unused imports around lines that were touched. No meaningful code changes were performed as part of the code move. MozReview-Commit-ID: 96mGWUJ7oLb
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/mach_commands.py
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -3,32 +3,32 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, unicode_literals
 
 import getpass
 import json
 import logging
 import os
-import platform
 import subprocess
 import sys
 import time
 import which
 
 from collections import (
     namedtuple,
     OrderedDict,
 )
 
 try:
     import psutil
 except Exception:
     psutil = None
 
+from mach.mixin.logging import LoggingMixin
 from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
 
 import mozpack.path as mozpath
 
 from ..base import MozbuildObject
 
 from ..testing import install_test_files
 
@@ -480,16 +480,285 @@ class BuildMonitor(MozbuildObject):
         except which.WhichError:
             pass
         except ValueError as e:
             self.log(logging.WARNING, 'ccache', {'msg': str(e)}, '{msg}')
 
         return ccache_stats
 
 
+class TerminalLoggingHandler(logging.Handler):
+    """Custom logging handler that works with terminal window dressing.
+
+    This class should probably live elsewhere, like the mach core. Consider
+    this a proving ground for its usefulness.
+    """
+    def __init__(self):
+        logging.Handler.__init__(self)
+
+        self.fh = sys.stdout
+        self.footer = None
+
+    def flush(self):
+        self.acquire()
+
+        try:
+            self.fh.flush()
+        finally:
+            self.release()
+
+    def emit(self, record):
+        msg = self.format(record)
+
+        self.acquire()
+
+        try:
+            if self.footer:
+                self.footer.clear()
+
+            self.fh.write(msg)
+            self.fh.write('\n')
+
+            if self.footer:
+                self.footer.draw()
+
+            # If we don't flush, the footer may not get drawn.
+            self.fh.flush()
+        finally:
+            self.release()
+
+
+class Footer(object):
+    """Handles display of a footer in a terminal.
+
+    This class implements the functionality common to all mach commands
+    that render a footer.
+    """
+
+    def __init__(self, terminal):
+        # terminal is a blessings.Terminal.
+        self._t = terminal
+        self._fh = sys.stdout
+
+    def clear(self):
+        """Removes the footer from the current terminal."""
+        self._fh.write(self._t.move_x(0))
+        self._fh.write(self._t.clear_eol())
+
+    def write(self, parts):
+        """Write some output in the footer, accounting for terminal width.
+
+        parts is a list of 2-tuples of (encoding_function, input).
+        None means no encoding."""
+
+        # We don't want to write more characters than the current width of the
+        # terminal otherwise wrapping may result in weird behavior. We can't
+        # simply truncate the line at terminal width characters because a)
+        # non-viewable escape characters count towards the limit and b) we
+        # don't want to truncate in the middle of an escape sequence because
+        # subsequent output would inherit the escape sequence.
+        max_width = self._t.width
+        written = 0
+        write_pieces = []
+        for part in parts:
+            try:
+                func, part = part
+                encoded = getattr(self._t, func)(part)
+            except ValueError:
+                encoded = part
+
+            len_part = len(part)
+            len_spaces = len(write_pieces)
+            if written + len_part + len_spaces > max_width:
+                write_pieces.append(part[0:max_width - written - len_spaces])
+                written += len_part
+                break
+
+            write_pieces.append(encoded)
+            written += len_part
+
+        with self._t.location():
+            self._t.move(self._t.height-1,0)
+            self._fh.write(' '.join(write_pieces))
+
+
+class BuildProgressFooter(Footer):
+    """Handles display of a build progress indicator in a terminal.
+
+    When mach builds inside a blessings-supported terminal, it will render
+    progress information collected from a BuildMonitor. This class converts the
+    state of BuildMonitor into terminal output.
+    """
+
+    def __init__(self, terminal, monitor):
+        Footer.__init__(self, terminal)
+        self.tiers = monitor.tiers.tier_status.viewitems()
+
+    def draw(self):
+        """Draws this footer in the terminal."""
+
+        if not self.tiers:
+            return
+
+        # The drawn terminal looks something like:
+        # TIER: static export libs tools
+
+        parts = [('bold', 'TIER:')]
+        append = parts.append
+        for tier, status in self.tiers:
+            if status is None:
+                append(tier)
+            elif status == 'finished':
+                append(('green', tier))
+            else:
+                append(('underline_yellow', tier))
+
+        self.write(parts)
+
+
+
+class OutputManager(LoggingMixin):
+    """Handles writing job output to a terminal or log."""
+
+    def __init__(self, log_manager, footer):
+        self.populate_logger()
+
+        self.footer = None
+        terminal = log_manager.terminal
+
+        # TODO convert terminal footer to config file setting.
+        if not terminal or os.environ.get('MACH_NO_TERMINAL_FOOTER', None):
+            return
+        if os.environ.get('INSIDE_EMACS', None):
+            return
+
+        self.t = terminal
+        self.footer = footer
+
+        self._handler = TerminalLoggingHandler()
+        self._handler.setFormatter(log_manager.terminal_formatter)
+        self._handler.footer = self.footer
+
+        old = log_manager.replace_terminal_handler(self._handler)
+        self._handler.level = old.level
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self.footer:
+            self.footer.clear()
+            # Prevents the footer from being redrawn if logging occurs.
+            self._handler.footer = None
+
+    def write_line(self, line):
+        if self.footer:
+            self.footer.clear()
+
+        print(line)
+
+        if self.footer:
+            self.footer.draw()
+
+    def refresh(self):
+        if not self.footer:
+            return
+
+        self.footer.clear()
+        self.footer.draw()
+
+
+class BuildOutputManager(OutputManager):
+    """Handles writing build output to a terminal, to logs, etc."""
+
+    def __init__(self, log_manager, monitor, footer):
+        self.monitor = monitor
+        OutputManager.__init__(self, log_manager, footer)
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        OutputManager.__exit__(self, exc_type, exc_value, traceback)
+
+        # Ensure the resource monitor is stopped because leaving it running
+        # could result in the process hanging on exit because the resource
+        # collection child process hasn't been told to stop.
+        self.monitor.stop_resource_recording()
+
+
+    def on_line(self, line):
+        warning, state_changed, relevant = self.monitor.on_line(line)
+
+        if relevant:
+            self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
+        elif state_changed:
+            have_handler = hasattr(self, 'handler')
+            if have_handler:
+                self.handler.acquire()
+            try:
+                self.refresh()
+            finally:
+                if have_handler:
+                    self.handler.release()
+
+
+class StaticAnalysisFooter(Footer):
+    """Handles display of a static analysis progress indicator in a terminal.
+    """
+
+    def __init__(self, terminal, monitor):
+        Footer.__init__(self, terminal)
+        self.monitor = monitor
+
+    def draw(self):
+        """Draws this footer in the terminal."""
+
+        monitor = self.monitor
+        total = monitor.num_files
+        processed = monitor.num_files_processed
+        percent = '(%.2f%%)' % (processed * 100.0 / total)
+        parts = [
+            ('dim', 'Processing'),
+            ('yellow', str(processed)),
+            ('dim', 'of'),
+            ('yellow', str(total)),
+            ('dim', 'files'),
+            ('green', percent)
+        ]
+        if monitor.current_file:
+            parts.append(('bold', monitor.current_file))
+
+        self.write(parts)
+
+
+class StaticAnalysisOutputManager(OutputManager):
+    """Handles writing static analysis output to a terminal."""
+
+    def __init__(self, log_manager, monitor, footer):
+        self.monitor = monitor
+        OutputManager.__init__(self, log_manager, footer)
+
+    def on_line(self, line):
+        warning, relevant = self.monitor.on_line(line)
+
+        if warning:
+            self.log(logging.INFO, 'compiler_warning', warning,
+                'Warning: {flag} in {filename}: {message}')
+
+        if relevant:
+            self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
+        else:
+            have_handler = hasattr(self, 'handler')
+            if have_handler:
+                self.handler.acquire()
+            try:
+                self.refresh()
+            finally:
+                if have_handler:
+                    self.handler.release()
+
+
 class CCacheStats(object):
     """Holds statistics from ccache.
 
     Instances can be subtracted from each other to obtain differences.
     print() or str() the object to show a ``ccache -s`` like output
     of the captured stats.
 
     """
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1,17 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
 import collections
-import errno
 import hashlib
 import itertools
 import json
 import logging
 import operator
 import os
 import subprocess
 import sys
@@ -24,28 +23,23 @@ from mach.decorators import (
     CommandArgument,
     CommandArgumentGroup,
     CommandProvider,
     Command,
     SettingsProvider,
     SubCommand,
 )
 
-from mach.mixin.logging import LoggingMixin
-
 from mach.main import Mach
 
 from mozbuild.base import (
     BuildEnvironmentNotFoundException,
     MachCommandBase,
     MachCommandConditions as conditions,
     MozbuildObject,
-    MozconfigFindException,
-    MozconfigLoadException,
-    ObjdirMismatchException,
 )
 from mozbuild.util import ensureParentDir
 
 from mozbuild.backend import (
     backends,
     get_backend_class,
 )
 from mozbuild.shellutil import quote as shell_quote
@@ -86,227 +80,16 @@ If you feel this message is not appropri
 please file a Core :: Build Config bug at
 https://bugzilla.mozilla.org/enter_bug.cgi?product=Core&component=Build%20Config
 and tell us about your machine and build configuration so we can adjust the
 warning heuristic.
 ===================
 '''
 
 
-class TerminalLoggingHandler(logging.Handler):
-    """Custom logging handler that works with terminal window dressing.
-
-    This class should probably live elsewhere, like the mach core. Consider
-    this a proving ground for its usefulness.
-    """
-    def __init__(self):
-        logging.Handler.__init__(self)
-
-        self.fh = sys.stdout
-        self.footer = None
-
-    def flush(self):
-        self.acquire()
-
-        try:
-            self.fh.flush()
-        finally:
-            self.release()
-
-    def emit(self, record):
-        msg = self.format(record)
-
-        self.acquire()
-
-        try:
-            if self.footer:
-                self.footer.clear()
-
-            self.fh.write(msg)
-            self.fh.write('\n')
-
-            if self.footer:
-                self.footer.draw()
-
-            # If we don't flush, the footer may not get drawn.
-            self.fh.flush()
-        finally:
-            self.release()
-
-
-class Footer(object):
-    """Handles display of a footer in a terminal.
-
-    This class implements the functionality common to all mach commands
-    that render a footer.
-    """
-
-    def __init__(self, terminal):
-        # terminal is a blessings.Terminal.
-        self._t = terminal
-        self._fh = sys.stdout
-
-    def clear(self):
-        """Removes the footer from the current terminal."""
-        self._fh.write(self._t.move_x(0))
-        self._fh.write(self._t.clear_eol())
-
-    def write(self, parts):
-        """Write some output in the footer, accounting for terminal width.
-
-        parts is a list of 2-tuples of (encoding_function, input).
-        None means no encoding."""
-
-        # We don't want to write more characters than the current width of the
-        # terminal otherwise wrapping may result in weird behavior. We can't
-        # simply truncate the line at terminal width characters because a)
-        # non-viewable escape characters count towards the limit and b) we
-        # don't want to truncate in the middle of an escape sequence because
-        # subsequent output would inherit the escape sequence.
-        max_width = self._t.width
-        written = 0
-        write_pieces = []
-        for part in parts:
-            try:
-                func, part = part
-                encoded = getattr(self._t, func)(part)
-            except ValueError:
-                encoded = part
-
-            len_part = len(part)
-            len_spaces = len(write_pieces)
-            if written + len_part + len_spaces > max_width:
-                write_pieces.append(part[0:max_width - written - len_spaces])
-                written += len_part
-                break
-
-            write_pieces.append(encoded)
-            written += len_part
-
-        with self._t.location():
-            self._t.move(self._t.height-1,0)
-            self._fh.write(' '.join(write_pieces))
-
-
-class BuildProgressFooter(Footer):
-    """Handles display of a build progress indicator in a terminal.
-
-    When mach builds inside a blessings-supported terminal, it will render
-    progress information collected from a BuildMonitor. This class converts the
-    state of BuildMonitor into terminal output.
-    """
-
-    def __init__(self, terminal, monitor):
-        Footer.__init__(self, terminal)
-        self.tiers = monitor.tiers.tier_status.viewitems()
-
-    def draw(self):
-        """Draws this footer in the terminal."""
-
-        if not self.tiers:
-            return
-
-        # The drawn terminal looks something like:
-        # TIER: static export libs tools
-
-        parts = [('bold', 'TIER:')]
-        append = parts.append
-        for tier, status in self.tiers:
-            if status is None:
-                append(tier)
-            elif status == 'finished':
-                append(('green', tier))
-            else:
-                append(('underline_yellow', tier))
-
-        self.write(parts)
-
-
-class OutputManager(LoggingMixin):
-    """Handles writing job output to a terminal or log."""
-
-    def __init__(self, log_manager, footer):
-        self.populate_logger()
-
-        self.footer = None
-        terminal = log_manager.terminal
-
-        # TODO convert terminal footer to config file setting.
-        if not terminal or os.environ.get('MACH_NO_TERMINAL_FOOTER', None):
-            return
-        if os.environ.get('INSIDE_EMACS', None):
-            return
-
-        self.t = terminal
-        self.footer = footer
-
-        self._handler = TerminalLoggingHandler()
-        self._handler.setFormatter(log_manager.terminal_formatter)
-        self._handler.footer = self.footer
-
-        old = log_manager.replace_terminal_handler(self._handler)
-        self._handler.level = old.level
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        if self.footer:
-            self.footer.clear()
-            # Prevents the footer from being redrawn if logging occurs.
-            self._handler.footer = None
-
-    def write_line(self, line):
-        if self.footer:
-            self.footer.clear()
-
-        print(line)
-
-        if self.footer:
-            self.footer.draw()
-
-    def refresh(self):
-        if not self.footer:
-            return
-
-        self.footer.clear()
-        self.footer.draw()
-
-class BuildOutputManager(OutputManager):
-    """Handles writing build output to a terminal, to logs, etc."""
-
-    def __init__(self, log_manager, monitor, footer):
-        self.monitor = monitor
-        OutputManager.__init__(self, log_manager, footer)
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        OutputManager.__exit__(self, exc_type, exc_value, traceback)
-
-        # Ensure the resource monitor is stopped because leaving it running
-        # could result in the process hanging on exit because the resource
-        # collection child process hasn't been told to stop.
-        self.monitor.stop_resource_recording()
-
-
-    def on_line(self, line):
-        warning, state_changed, relevant = self.monitor.on_line(line)
-
-        if relevant:
-            self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
-        elif state_changed:
-            have_handler = hasattr(self, 'handler')
-            if have_handler:
-                self.handler.acquire()
-            try:
-                self.refresh()
-            finally:
-                if have_handler:
-                    self.handler.release()
-
-
 class StoreDebugParamsAndWarnAction(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
         sys.stderr.write('The --debugparams argument is deprecated. Please ' +
                          'use --debugger-args instead.\n\n')
         setattr(namespace, self.dest, values)
 
 
 @CommandProvider
@@ -382,18 +165,21 @@ class Build(MachCommandBase):
           libraries and executables (binaries).
 
         * faster - builds JavaScript, XUL, CSS, etc files.
 
         "binaries" and "faster" almost fully complement each other. However,
         there are build actions not captured by either. If things don't appear to
         be rebuilding, perform a vanilla `mach build` to rebuild the world.
         """
-        import which
-        from mozbuild.controller.building import BuildMonitor
+        from mozbuild.controller.building import (
+            BuildMonitor,
+            BuildOutputManager,
+            BuildProgressFooter,
+        )
         from mozbuild.util import (
             mkdir,
             resolve_target_to_make,
         )
 
         self.log_manager.register_structured_logger(logging.getLogger('mozbuild'))
 
         warnings_path = self._get_state_filename('warnings.json')
@@ -2116,72 +1902,16 @@ class StaticAnalysisMonitor(object):
                 self._current = os.path.relpath(filename, self._srcdir)
             else:
                 self._current = None
             self._processed = self._processed + 1
             return (warning, False)
         return (warning, True)
 
 
-class StaticAnalysisFooter(Footer):
-    """Handles display of a static analysis progress indicator in a terminal.
-    """
-
-    def __init__(self, terminal, monitor):
-        Footer.__init__(self, terminal)
-        self.monitor = monitor
-
-    def draw(self):
-        """Draws this footer in the terminal."""
-
-        monitor = self.monitor
-        total = monitor.num_files
-        processed = monitor.num_files_processed
-        percent = '(%.2f%%)' % (processed * 100.0 / total)
-        parts = [
-            ('dim', 'Processing'),
-            ('yellow', str(processed)),
-            ('dim', 'of'),
-            ('yellow', str(total)),
-            ('dim', 'files'),
-            ('green', percent)
-        ]
-        if monitor.current_file:
-            parts.append(('bold', monitor.current_file))
-
-        self.write(parts)
-
-
-class StaticAnalysisOutputManager(OutputManager):
-    """Handles writing static analysis output to a terminal."""
-
-    def __init__(self, log_manager, monitor, footer):
-        self.monitor = monitor
-        OutputManager.__init__(self, log_manager, footer)
-
-    def on_line(self, line):
-        warning, relevant = self.monitor.on_line(line)
-
-        if warning:
-            self.log(logging.INFO, 'compiler_warning', warning,
-                'Warning: {flag} in {filename}: {message}')
-
-        if relevant:
-            self.log(logging.INFO, 'build_output', {'line': line}, '{line}')
-        else:
-            have_handler = hasattr(self, 'handler')
-            if have_handler:
-                self.handler.acquire()
-            try:
-                self.refresh()
-            finally:
-                if have_handler:
-                    self.handler.release()
-
-
 @CommandProvider
 class StaticAnalysis(MachCommandBase):
     """Utilities for running C++ static analysis checks."""
 
     @Command('static-analysis', category='testing',
              description='Run C++ static analysis checks')
     def static_analysis(self):
         # If not arguments are provided, just print a help message.
@@ -2208,16 +1938,21 @@ class StaticAnalysis(MachCommandBase):
     @CommandArgument('--fix', '-f', default=False, action='store_true',
                      help='Try to autofix errors detected by clang-tidy checkers.')
     @CommandArgument('--header-filter', '-h-f', default='', metavar='header_filter',
                      help='Regular expression matching the names of the headers to '
                           'output diagnostics from. Diagnostics from the main file '
                           'of each translation unit are always displayed')
     def check(self, source=None, jobs=2, strip=1, verbose=False,
               checks='-*', fix=False, header_filter=''):
+        from mozbuild.controller.building import (
+            StaticAnalysisFooter,
+            StaticAnalysisOutputManager,
+        )
+
         self._set_log_level(verbose)
         rc = self._build_compile_db(verbose=verbose)
         if rc != 0:
             return rc
 
         rc = self._build_export(jobs=jobs, verbose=verbose)
         if rc != 0:
             return rc