Bug 1302796 - Add integration between structured logging and lints, r=ahal
authorJames Graham <james@hoppipolla.co.uk>
Tue, 13 Sep 2016 14:18:41 +0100
changeset 357720 d3fc2851c3fc76920cd6759e8d50b6e698518ac5
parent 357719 96e13a1757524e0eada5ab197719e63e870dc725
child 357721 f06b81832d6fd8507a2591cc45771d3aadb4553e
push id1324
push usermtabara@mozilla.com
push dateMon, 16 Jan 2017 13:07:44 +0000
treeherdermozilla-release@a01c49833940 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1302796
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1302796 - Add integration between structured logging and lints, r=ahal MozReview-Commit-ID: K3tu0Zdg5go
python/mozlint/mozlint/types.py
python/mozlint/setup.py
python/mozlint/test/linters/structured.lint
python/mozlint/test/test_types.py
testing/mozbase/mozlog/mozlog/formatters/errorsummary.py
testing/mozbase/mozlog/mozlog/formatters/machformatter.py
testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
testing/mozbase/mozlog/mozlog/logtypes.py
testing/mozbase/mozlog/mozlog/structuredlog.py
testing/mozbase/mozlog/setup.py
tools/lint/docs/create.rst
--- a/python/mozlint/mozlint/types.py
+++ b/python/mozlint/mozlint/types.py
@@ -1,17 +1,21 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 import re
+import sys
 from abc import ABCMeta, abstractmethod
 
+from mozlog import get_default_logger, commandline, structuredlog
+from mozlog.reader import LogHandler
+
 from . import result
 from .pathutils import filterpaths
 
 
 class BaseType(object):
     """Abstract base class for all types of linters."""
     __metaclass__ = ABCMeta
     batch = False
@@ -96,14 +100,43 @@ class ExternalType(BaseType):
     """
     batch = True
 
     def _lint(self, files, linter, **lintargs):
         payload = linter['payload']
         return payload(files, **lintargs)
 
 
+class LintHandler(LogHandler):
+    def __init__(self, linter):
+        self.linter = linter
+        self.results = []
+
+    def lint(self, data):
+        self.results.append(result.from_linter(self.linter, **data))
+
+
+class StructuredLogType(BaseType):
+    batch = True
+
+    def _lint(self, files, linter, **lintargs):
+        payload = linter["payload"]
+        handler = LintHandler(linter)
+        logger = linter.get("logger")
+        if logger is None:
+            logger = get_default_logger()
+        if logger is None:
+            logger = structuredlog.StructuredLogger(linter["name"])
+            commandline.setup_logging(logger, {}, {"mach": sys.stdout})
+        logger.add_handler(handler)
+        try:
+            payload(files, logger, **lintargs)
+        except KeyboardInterrupt:
+            pass
+        return handler.results
+
 supported_types = {
     'string': StringType(),
     'regex': RegexType(),
     'external': ExternalType(),
+    'structured_log': StructuredLogType()
 }
 """Mapping of type string to an associated instance."""
--- a/python/mozlint/setup.py
+++ b/python/mozlint/setup.py
@@ -1,16 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from setuptools import setup
 
 VERSION = 0.1
-DEPS = []
+DEPS = ["mozlog>=3.4"]
 
 setup(
     name='mozlint',
     description='Framework for registering and running micro lints',
     license='MPL 2.0',
     author='Andrew Halberstadt',
     author_email='ahalberstadt@mozilla.com',
     url='',
new file mode 100644
--- /dev/null
+++ b/python/mozlint/test/linters/structured.lint
@@ -0,0 +1,28 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+def lint(files, logger, **kwargs):
+    for path in files:
+        with open(path, 'r') as fh:
+            for i, line in enumerate(fh.readlines()):
+                if 'foobar' in line:
+                    logger.lint_error(path=path,
+                                      lineno=i+1,
+                                      column=1,
+                                      rule="no-foobar")
+
+
+LINTER = {
+    'name': "StructuredLinter",
+    'description': "It's bad to have the string foobar in js files.",
+    'include': [
+        'files',
+    ],
+    'type': 'structured_log',
+    'extensions': ['.js', '.jsm'],
+    'payload': lint,
+}
--- a/python/mozlint/test/test_types.py
+++ b/python/mozlint/test/test_types.py
@@ -12,17 +12,17 @@ from mozlint.result import ResultContain
 
 @pytest.fixture
 def path(filedir):
     def _path(name):
         return os.path.join(filedir, name)
     return _path
 
 
-@pytest.fixture(params=['string.lint', 'regex.lint', 'external.lint'])
+@pytest.fixture(params=['string.lint', 'regex.lint', 'external.lint', 'structured.lint'])
 def linter(lintdir, request):
     return os.path.join(lintdir, request.param)
 
 
 def test_linter_types(lint, linter, files, path):
     lint.read(linter)
     result = lint.roll(files)
     assert isinstance(result, dict)
--- a/testing/mozbase/mozlog/mozlog/formatters/errorsummary.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/errorsummary.py
@@ -48,8 +48,20 @@ class ErrorSummaryFormatter(BaseFormatte
         return self._output("log", data)
 
     def crash(self, item):
         data = {"test": item.get("test"),
                 "signature": item["signature"],
                 "stackwalk_stdout": item.get("stackwalk_stdout"),
                 "stackwalk_stderr": item.get("stackwalk_stderr")}
         return self._output("crash", data)
+
+    def lint(self, item):
+        data = {
+            "level": item["level"],
+            "path": item["path"],
+            "message": item["message"],
+            "lineno": item["lineno"],
+            "column": item.get("column"),
+            "rule": item.get("rule"),
+            "linter": item.get("linter")
+        }
+        self._output("lint", data)
--- a/testing/mozbase/mozlog/mozlog/formatters/machformatter.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/machformatter.py
@@ -351,16 +351,35 @@ class MachFormatter(base.BaseFormatter):
         else:
             rv = "%s %s" % (level, data["message"])
 
         if "stack" in data:
             rv += "\n%s" % data["stack"]
 
         return rv
 
+    def lint(self, data):
+        term = self.terminal if self.terminal is not None else NullTerminal()
+        fmt = "{path}  {c1}{lineno}{column}  {c2}{level}{normal}  {message}  {c1}{rule}({linter}){normal}"
+        message = fmt.format(
+            path=data["path"],
+            normal=term.normal,
+            c1=term.grey,
+            c2=term.red if data["level"] == 'error' else term.yellow,
+            lineno=str(data["lineno"]),
+            column=(":" + str(data["column"])) if data.get("column") else "",
+            level=data["level"],
+            message=data["message"],
+            rule='{} '.format(data["rule"]) if data.get("rule") else "",
+            linter=data["linter"].lower() if data.get("linter") else "",
+        )
+
+        return message
+
+
     def _get_subtest_data(self, data):
         test = self._get_test_id(data)
         return self.status_buffer.get(test, {"count": 0, "unexpected": [], "pass": 0})
 
     def _time(self, data):
         entry_time = data["time"]
         if self.write_interval and self.last_time is not None:
             t = entry_time - self.last_time
--- a/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
@@ -228,8 +228,14 @@ class TbplFormatter(BaseFormatter):
 
     @output_subtests
     def valgrind_error(self, data):
         rv = "TEST-UNEXPECTED-VALGRIND-ERROR | " + data['primary'] + "\n"
         for line in data['secondary']:
             rv = rv + line + "\n"
 
         return rv
+
+    def lint(self, data):
+        fmt = "TEST-UNEXPECTED-{level} | {path}:{lineno}{column} | {message} ({rule})"
+        data["column"] = ":%s" % data["column"] if data["column"] else ""
+        data['rule'] = data['rule'] or data['linter'] or ""
+        message.append(fmt.format(**data))
--- a/testing/mozbase/mozlog/mozlog/logtypes.py
+++ b/testing/mozbase/mozlog/mozlog/logtypes.py
@@ -152,23 +152,37 @@ class Status(DataType):
 
 class SubStatus(Status):
     allowed = ["PASS", "FAIL", "ERROR", "TIMEOUT", "ASSERT", "NOTRUN", "SKIP"]
 
 class Dict(DataType):
     def convert(self, data):
         return dict(data)
 
+
 class List(DataType):
     def __init__(self, name, item_type, default=no_default, optional=False):
         DataType.__init__(self, name, default, optional)
         self.item_type = item_type(None)
 
     def convert(self, data):
         return [self.item_type.convert(item) for item in data]
 
 class Int(DataType):
     def convert(self, data):
         return int(data)
 
+
 class Any(DataType):
     def convert(self, data):
         return data
+
+
+class Tuple(DataType):
+    def __init__(self, name, item_types, default=no_default, optional=False):
+        DataType.__init__(self, name, default, optional)
+        self.item_types = item_types
+
+    def convert(self, data):
+        if len(data) != len(self.item_types):
+            raise ValueError("Expected %i items got %i" % (len(self.item_types), len(data)))
+        return tuple(item_type.convert(value)
+                     for item_type, value in zip(self.item_types, data))
--- a/testing/mozbase/mozlog/mozlog/structuredlog.py
+++ b/testing/mozbase/mozlog/mozlog/structuredlog.py
@@ -6,17 +6,17 @@ from __future__ import unicode_literals
 
 from multiprocessing import current_process
 from threading import current_thread, Lock
 import json
 import sys
 import time
 import traceback
 
-from logtypes import Unicode, TestId, Status, SubStatus, Dict, List, Int, Any
+from logtypes import Unicode, TestId, Status, SubStatus, Dict, List, Int, Any, Tuple
 from logtypes import log_action, convertor_registry
 
 """Structured Logging for recording test results.
 
 Allowed actions, and subfields:
   suite_start
       tests  - List of test names
 
@@ -88,16 +88,18 @@ def set_default_logger(default_logger):
     """
     global _default_logger_name
 
     _default_logger_name = default_logger.name
 
 log_levels = dict((k.upper(), v) for v, k in
                   enumerate(["critical", "error", "warning", "info", "debug"]))
 
+lint_levels = ["ERROR", "WARNING"]
+
 def log_actions():
     """Returns the set of actions implemented by mozlog."""
     return set(convertor_registry.keys())
 
 class LoggerState(object):
     def __init__(self):
         self.handlers = []
         self.running_tests = set()
@@ -433,21 +435,54 @@ def _log_func(level_name):
 :param exc_info: Either a boolean indicating whether to include a traceback
                  derived from sys.exc_info() or a three-item tuple in the
                  same format as sys.exc_info() containing exception information
                  to log.
 """ % level_name
     log.__name__ = str(level_name).lower()
     return log
 
+def _lint_func(level_name):
+    @log_action(Unicode("path"),
+                Unicode("message", default=""),
+                Int("lineno", default=0),
+                Int("column", default=None, optional=True),
+                Unicode("hint", default=None, optional=True),
+                Unicode("source", default=None, optional=True),
+                Unicode("rule", default=None, optional=True),
+                Tuple("lineoffset", (Int, Int), default=None, optional=True),
+                Unicode("linter", default=None, optional=True))
+    def lint(self, data):
+        data["level"] = level_name
+        self._log_data("lint", data)
+    lint.__doc__ = """Log an error resulting from a failed lint check
 
-# Create all the methods on StructuredLog for debug levels
+        :param linter: name of the linter that flagged this error
+        :param path: path to the file containing the error
+        :param message: text describing the error
+        :param lineno: line number that contains the error
+        :param column: column containing the error
+        :param hint: suggestion for fixing the error (optional)
+        :param source: source code context of the error (optional)
+        :param rule: name of the rule that was violated (optional)
+        :param lineoffset: denotes an error spans multiple lines, of the form
+                           (<lineno offset>, <num lines>) (optional)
+        """
+    lint.__name__ = str("lint_%s" % level_name)
+    return lint
+
+
+# Create all the methods on StructuredLog for log/lint levels
 for level_name in log_levels:
     setattr(StructuredLogger, level_name.lower(), _log_func(level_name))
 
+for level_name in lint_levels:
+    level_name = level_name.lower()
+    name = "lint_%s" % level_name
+    setattr(StructuredLogger, name, _lint_func(level_name))
 
 class StructuredLogFileLike(object):
     """Wrapper for file-like objects to redirect writes to logger
     instead. Each call to `write` becomes a single log entry of type `log`.
 
     When using this it is important that the callees i.e. the logging
     handlers do not themselves try to write to the wrapped file as this
     will cause infinite recursion.
--- a/testing/mozbase/mozlog/setup.py
+++ b/testing/mozbase/mozlog/setup.py
@@ -1,16 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from setuptools import setup, find_packages
 
 PACKAGE_NAME = 'mozlog'
-PACKAGE_VERSION = '3.3'
+PACKAGE_VERSION = '3.4'
 
 setup(name=PACKAGE_NAME,
       version=PACKAGE_VERSION,
       description="Robust log handling specialized for logging in the Mozilla universe",
       long_description="see http://mozbase.readthedocs.org/",
       author='Mozilla Automation and Testing Team',
       author_email='tools@lists.mozilla.org',
       url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase',
--- a/tools/lint/docs/create.rst
+++ b/tools/lint/docs/create.rst
@@ -24,27 +24,38 @@ Now ``no-eval.lint`` gets passed into :f
 Linter Types
 ------------
 
 There are three types of linters, though more may be added in the future.
 
 1. string - fails if substring is found
 2. regex - fails if regex matches
 3. external - fails if a python function returns a non-empty result list
+4. structured_log - fails if a mozlog logger emits any lint_error or lint_warning log messages
 
 As seen from the example above, string and regex linters are very easy to create, but they
 should be avoided if possible. It is much better to use a context aware linter for the language you
 are trying to lint. For example, use eslint to lint JavaScript files, use flake8 to lint python
 files, etc.
 
-Which brings us to the third and most interesting type of linter, external.  External linters call
-an arbitrary python function which is responsible for not only running the linter, but ensuring the
-results are structured properly. For example, an external type could shell out to a 3rd party
-linter, collect the output and format it into a list of :class:`ResultContainer` objects.
+Which brings us to the third and most interesting type of linter,
+external.  External linters call an arbitrary python function which is
+responsible for not only running the linter, but ensuring the results
+are structured properly. For example, an external type could shell out
+to a 3rd party linter, collect the output and format it into a list of
+:class:`ResultContainer` objects. The signature for this python
+function is ``lint(files, **kwargs)``, where ``files`` is a list of
+files to lint.
 
+Structured log linters are much like external linters, but suitable
+for cases where the linter code is using mozlog and emits
+``lint_error`` or ``lint_warning`` logging messages when the lint
+fails. This is recommended for writing novel gecko-specific lints. In
+this case the signature for lint functions is ``lint(files, logger,
+**kwargs)``.
 
 LINTER Definition
 -----------------
 
 Each ``.lint`` file must have a variable called LINTER which is a dict containing metadata about the
 linter. Here are the supported keys:
 
 * name - The name of the linter (required)
@@ -59,16 +70,20 @@ linter. Here are the supported keys:
 In addition to the above, some ``.lint`` files correspond to a single lint rule. For these, the
 following additional keys may be specified:
 
 * message - A string to print on infraction (optional)
 * hint - A string with a clue on how to fix the infraction (optional)
 * rule - An id string for the lint rule (optional)
 * level - The severity of the infraction, either 'error' or 'warning' (optional)
 
+For structured_log lints the following additional keys apply:
+
+* logger - A StructuredLog object to use for logging. If not supplied
+  one will be created (optional)
 
 Example
 -------
 
 Here is an example of an external linter that shells out to the python flake8 linter:
 
 .. code-block:: python