Bug 1424287 - [mozlog] added color dictionary and TerminalColors class r=jgraham
authorvrinda <v.singhal373@gmail.com>
Thu, 21 Mar 2019 11:59:55 +0000
changeset 465399 6f59938c2690754fbc974587b3b267f79fe00ef1
parent 465398 5841b4e9959374c8d594e50bf1141d36edb5ced8
child 465400 6b83f9824d752a071d562c078ff7b92b03eb1a8d
push id35738
push userccoroiu@mozilla.com
push dateThu, 21 Mar 2019 21:59:09 +0000
treeherdermozilla-central@7eb8e627961c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgraham
bugs1424287
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1424287 - [mozlog] added color dictionary and TerminalColors class r=jgraham The default color dictionary is implemented, to avoid formattor specifying the colors discretely Differential Revision: https://phabricator.services.mozilla.com/D24196
testing/mozbase/mozlog/mozlog/formatters/machformatter.py
--- a/testing/mozbase/mozlog/mozlog/formatters/machformatter.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/machformatter.py
@@ -1,66 +1,87 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
 from __future__ import absolute_import
 
+from mozterm import Terminal
 import time
-
-from mozterm import Terminal
-
 from . import base
 from .process import strstatus
 from .tbplformatter import TbplFormatter
 from ..handlers import SummaryHandler
 import six
 from functools import reduce
 
+color_dict = {
+    'log_test_status_fail': 'yellow',
+    'log_process_output': 'blue',
+    'log_test_status_pass': 'green',
+    'log_test_status_unexpected_fail': 'red',
+    'time': 'cyan',
+    'action': 'yellow',
+    'pid': 'cyan',
+    'heading': 'bold_yellow',
+    'error': 'red',
+    'warning': 'yellow',
+    'bold': 'bold',
+    'grey': 'grey',
+    'normal': 'normal',
+    'dim': 'dim'
+}
+
 
 def format_seconds(total):
     """Format number of seconds to MM:SS.DD form."""
     minutes, seconds = divmod(total, 60)
     return '%2d:%05.2f' % (minutes, seconds)
 
 
+class TerminalColors(object):
+    def __init__(self, term, color_dict):
+        for key, value in color_dict.items():
+            setattr(self, key, getattr(term, value))
+
+
 class MachFormatter(base.BaseFormatter):
 
     def __init__(self, start_time=None, write_interval=False, write_times=True,
                  terminal=None, disable_colors=False, summary_on_shutdown=False,
                  verbose=False, enable_screenshot=False, **kwargs):
         super(MachFormatter, self).__init__(**kwargs)
 
         if start_time is None:
             start_time = time.time()
         start_time = int(start_time * 1000)
         self.start_time = start_time
         self.write_interval = write_interval
         self.write_times = write_times
         self.status_buffer = {}
         self.has_unexpected = {}
         self.last_time = None
-        self.term = Terminal(disable_styling=disable_colors)
+        self.color_formatter = TerminalColors(
+            Terminal(disable_styling=disable_colors), color_dict)
         self.verbose = verbose
         self._known_pids = set()
         self.tbpl_formatter = None
         self.enable_screenshot = enable_screenshot
-
         self.summary = SummaryHandler()
         self.summary_on_shutdown = summary_on_shutdown
 
     def __call__(self, data):
         self.summary(data)
 
         s = super(MachFormatter, self).__call__(data)
         if s is None:
             return
 
-        time = self.term.dim_blue(format_seconds(self._time(data)))
+        time = self.color_formatter.time(format_seconds(self._time(data)))
         return "%s %s\n" % (time, s)
 
     def _get_test_id(self, data):
         test_id = data.get("test")
         if isinstance(test_id, list):
             test_id = tuple(test_id)
         return test_id
 
@@ -70,64 +91,71 @@ class MachFormatter(base.BaseFormatter):
 
         if isinstance(test_id, tuple):
             return "".join(test_id)
 
         assert False, "unexpected test_id"
 
     def suite_start(self, data):
         num_tests = reduce(lambda x, y: x + len(y), six.itervalues(data['tests']), 0)
-        action = self.term.yellow(data['action'].upper())
+        action = self.color_formatter.action(data['action'].upper())
         name = ""
         if 'name' in data:
             name = " %s -" % (data['name'],)
         return "%s:%s running %i tests" % (action, name, num_tests)
 
     def suite_end(self, data):
-        action = self.term.yellow(data['action'].upper())
+        action = self.color_formatter.action(data['action'].upper())
         rv = [action]
         if not self.summary_on_shutdown:
-            rv.append(self._format_suite_summary(self.summary.current_suite, self.summary.current))
+            rv.append(
+                self._format_suite_summary(
+                    self.summary.current_suite,
+                    self.summary.current))
         return "\n".join(rv)
 
     def _format_expected(self, status, expected):
         if status == expected:
-            color = self.term.green
+            color = self.color_formatter.log_test_status_pass
             if expected not in ("PASS", "OK"):
-                color = self.term.yellow
+                color = self.color_formatter.log_test_status_fail
                 status = "EXPECTED-%s" % status
         else:
-            color = self.term.red
+            color = self.color_formatter.log_test_status_pass
             if status in ("PASS", "OK"):
                 status = "UNEXPECTED-%s" % status
         return color(status)
 
     def _format_status(self, test, data):
         name = data.get("subtest", test)
         rv = "%s %s" % (self._format_expected(
             data["status"], data.get("expected", data["status"])), name)
         if "message" in data:
             rv += " - %s" % data["message"]
         if "stack" in data:
             rv += self._format_stack(data["stack"])
         return rv
 
     def _format_stack(self, stack):
-        return "\n%s\n" % self.term.dim(stack.strip("\n"))
+        return "\n%s\n" % self.color_formatter.dim(stack.strip("\n"))
 
     def _format_suite_summary(self, suite, summary):
         count = summary['counts']
         logs = summary['unexpected_logs']
 
-        rv = ["", self.term.yellow(suite), self.term.yellow("~" * len(suite))]
+        rv = [
+            "",
+            self.color_formatter.log_test_status_fail(suite),
+            self.color_formatter.log_test_status_fail(
+                "~" * len(suite))]
 
         # Format check counts
         checks = self.summary.aggregate('count', count)
-        rv.append("Ran {} checks ({})".format(sum(checks.values()),
-                  ', '.join(['{} {}s'.format(v, k) for k, v in sorted(checks.items()) if v])))
+        rv.append("Ran {} checks ({})".format(sum(checks.values()), ', '.join(
+            ['{} {}s'.format(v, k) for k, v in sorted(checks.items()) if v])))
 
         # Format expected counts
         checks = self.summary.aggregate('expected', count, include_skip=False)
         rv.append("Expected results: {}".format(sum(checks.values())))
 
         # Format skip counts
         skip_tests = count["test"]["expected"]["skip"]
         skip_subtests = count["subtest"]["expected"]["skip"]
@@ -147,38 +175,39 @@ class MachFormatter(base.BaseFormatter):
                     continue
                 status_str = ", ".join(["{} {}".format(n, s)
                                         for s, n in sorted(count[key]['unexpected'].items())])
                 rv.append("  {}: {} ({})".format(
                           key, sum(count[key]['unexpected'].values()), status_str))
 
         # Format status
         if not any(count[key]["unexpected"] for key in ('test', 'subtest', 'assert')):
-            rv.append(self.term.green("OK"))
+            rv.append(self.color_formatter.log_test_status_pass("OK"))
         else:
             heading = "Unexpected Results"
-            rv.extend(["", self.term.yellow(heading), self.term.yellow("-" * len(heading))])
+            rv.extend(["", self.color_formatter.heading(heading),
+                       self.color_formatter.heading("-" * len(heading))])
             if count['subtest']['count']:
                 for test_id, results in logs.items():
                     test = self._get_file_name(test_id)
-                    rv.append(self.term.bold(test))
+                    rv.append(self.color_formatter.bold(test))
                     for data in results:
                         rv.append("  %s" % self._format_status(test, data).rstrip())
             else:
                 for test_id, results in logs.items():
                     test = self._get_file_name(test_id)
                     assert len(results) == 1
                     data = results[0]
                     assert "subtest" not in data
                     rv.append(self._format_status(test, data).rstrip())
 
         return "\n".join(rv)
 
     def test_start(self, data):
-        action = self.term.yellow(data['action'].upper())
+        action = self.color_formatter.action(data['action'].upper())
         return "%s: %s" % (action, self._get_test_id(data))
 
     def test_end(self, data):
         subtests = self._get_subtest_data(data)
 
         if "expected" in data:
             parent_unexpected = True
             expected_str = ", expected %s" % data["expected"]
@@ -210,19 +239,19 @@ class MachFormatter(base.BaseFormatter):
                 if "stack" in data:
                     rv += self._format_stack(data["stack"])
             elif not self.verbose:
                 rv += "\n"
                 for d in unexpected:
                     rv += self._format_status(data['test'], d)
 
         if "expected" not in data and not bool(subtests['unexpected']):
-            color = self.term.green
+            color = self.color_formatter.log_test_status_pass
         else:
-            color = self.term.red
+            color = self.color_formatter.log_test_status_pass
 
         action = color(data['action'].upper())
         rv = "%s: %s" % (action, rv)
         if has_screenshots and self.enable_screenshot:
             if self.tbpl_formatter is None:
                 self.tbpl_formatter = TbplFormatter()
             # Create TBPL-like output that can be pasted into the reftest analyser
             rv = "\n".join((rv, self.tbpl_formatter.test_end(data)))
@@ -233,28 +262,28 @@ class MachFormatter(base.BaseFormatter):
         for line in data['secondary']:
             rv = rv + line + "\n"
 
         return rv
 
     def lsan_leak(self, data):
         allowed = data.get("allowed_match")
         if allowed:
-            prefix = self.term.yellow("FAIL")
+            prefix = self.color_formatter.log_test_status_fail("FAIL")
         else:
-            prefix = self.term.red("UNEXPECTED-FAIL")
+            prefix = self.color_formatter.log_test_status_unexpected_fail("UNEXPECTED-FAIL")
 
         return "%s LeakSanitizer: leak at %s" % (prefix, ", ".join(data["frames"]))
 
     def lsan_summary(self, data):
         allowed = data.get("allowed", False)
         if allowed:
-            prefix = self.term.yellow("WARNING")
+            prefix = self.color_formatter.warning("WARNING")
         else:
-            prefix = self.term.red("ERROR")
+            prefix = self.color_formatter.error("ERROR")
 
         return ("%s | LeakSanitizer | "
                 "SUMMARY: AddressSanitizer: %d byte(s) leaked in %d allocation(s)." %
                 (prefix, data["bytes"], data["allocations"]))
 
     def mozleak_object(self, data):
         data_log = data.copy()
         data_log["level"] = "INFO"
@@ -270,31 +299,34 @@ class MachFormatter(base.BaseFormatter):
                 data_log["level"] = "INFO"
                 data_log["message"] = ("leakcheck: %s deliberate crash and thus no leak log\n"
                                        % data["process"])
                 return self.log(data_log)
             if data.get("ignore_missing", False):
                 return ("%s ignoring missing output line for total leaks\n" %
                         data["process"])
 
-            status = self.term.red("FAIL")
+            status = self.color_formatter.log_test_status_pass("FAIL")
             return ("%s leakcheck: "
                     "%s missing output line for total leaks!\n" %
                     (status, data["process"]))
 
         if data["bytes"] == 0:
-            return ("%s leakcheck: %s no leaks detected!\n" %
-                    (self.term.green("PASS"), data["process"]))
+            return (
+                "%s leakcheck: %s no leaks detected!\n" %
+                (self.color_formatter.log_test_status_pass("PASS"),
+                    data["process"]))
 
         message = "leakcheck: %s %d bytes leaked\n" % (data["process"], data["bytes"])
 
         # data["bytes"] will include any expected leaks, so it can be off
         # by a few thousand bytes.
         failure = data["bytes"] > data["threshold"]
-        status = self.term.red("UNEXPECTED-FAIL") if failure else self.term.yellow("FAIL")
+        status = self.color_formatter.log_test_status_pass(
+            "UNEXPECTED-FAIL") if failure else self.color_formatter.log_test_status_fail("FAIL")
         return "%s %s\n" % (status, message)
 
     def test_status(self, data):
         test = self._get_test_id(data)
         if test not in self.status_buffer:
             self.status_buffer[test] = {"count": 0, "unexpected": 0, "pass": 0}
         self.status_buffer[test]["count"] += 1
 
@@ -311,27 +343,27 @@ class MachFormatter(base.BaseFormatter):
             return
 
         if data["min_expected"] != data["max_expected"]:
             expected = "%i to %i" % (data["min_expected"],
                                      data["max_expected"])
         else:
             expected = "%i" % data["min_expected"]
 
-        action = self.term.red("ASSERT")
+        action = self.color_formatter.log_test_status_pass("ASSERT")
         return "%s: Assertion count %i, expected %s assertions\n" % (
-                action, data["count"], expected)
+            action, data["count"], expected)
 
     def process_output(self, data):
         rv = []
 
         pid = data['process']
         if pid.isdigit():
             pid = 'pid:%s' % pid
-        pid = self.term.dim_cyan(pid)
+        pid = self.color_formatter.pid(pid)
 
         if "command" in data and data["process"] not in self._known_pids:
             self._known_pids.add(data["process"])
             rv.append('%s Full command: %s' % (pid, data["command"]))
 
         rv.append('%s %s' % (pid, data["data"]))
         return "\n".join(rv)
 
@@ -361,73 +393,76 @@ class MachFormatter(base.BaseFormatter):
 
         if data.get("stackwalk_errors"):
             rv.extend(data.get("stackwalk_errors"))
 
         rv = "\n".join(rv)
         if not rv[-1] == "\n":
             rv += "\n"
 
-        action = self.term.red(data['action'].upper())
+        action = self.color_formatter.action(data['action'].upper())
         return "%s: %s" % (action, rv)
 
     def process_start(self, data):
         rv = "Started process `%s`" % data['process']
         desc = data.get('command')
         if desc:
             rv = '%s (%s)' % (rv, desc)
         return rv
 
     def process_exit(self, data):
         return "%s: %s" % (data['process'], strstatus(data['exitcode']))
 
     def log(self, data):
         level = data.get("level").upper()
 
         if level in ("CRITICAL", "ERROR"):
-            level = self.term.red(level)
+            level = self.color_formatter.error(level)
         elif level == "WARNING":
-            level = self.term.yellow(level)
+            level = self.color_formatter.warning(level)
         elif level == "INFO":
-            level = self.term.blue(level)
+            level = self.color_formatter.log_process_output(level)
 
         if data.get('component'):
             rv = " ".join([data["component"], level, data["message"]])
         else:
             rv = "%s %s" % (level, data["message"])
 
         if "stack" in data:
             rv += "\n%s" % data["stack"]
 
         return rv
 
     def lint(self, data):
         fmt = "{path}  {c1}{lineno}{column}  {c2}{level}{normal}  {message}" \
               "  {c1}{rule}({linter}){normal}"
         message = fmt.format(
             path=data["path"],
-            normal=self.term.normal,
-            c1=self.term.grey,
-            c2=self.term.red if data["level"] == 'error' else self.term.yellow,
+            normal=self.color_formatter.normal,
+            c1=self.color_formatter.grey,
+            c2=self.color_formatter.error if data["level"] == 'error' else (
+                self.color_formatter.log_test_status_fail),
             lineno=str(data["lineno"]),
             column=(":" + str(data["column"])) if data.get("column") else "",
             level=data["level"],
             message=data["message"],
             rule='{} '.format(data["rule"]) if data.get("rule") else "",
             linter=data["linter"].lower() if data.get("linter") else "",
         )
 
         return message
 
     def shutdown(self, data):
         if not self.summary_on_shutdown:
             return
 
         heading = "Overall Summary"
-        rv = ["", self.term.bold_yellow(heading), self.term.bold_yellow("=" * len(heading))]
+        rv = [
+            "", self.color_formatter.heading(heading), self.color_formatter.heading(
+                "=" * len(heading))]
         for suite, summary in self.summary:
             rv.append(self._format_suite_summary(suite, summary))
         return "\n".join(rv)
 
     def _get_subtest_data(self, data):
         test = self._get_test_id(data)
         return self.status_buffer.get(test, {"count": 0, "unexpected": 0, "pass": 0})