Bug 1270506 - [mozlint] Add python flake8 linter, r=smacleod
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 05 May 2016 17:21:12 -0400
changeset 297663 971d076cfe29ca368dfb68397a84c89fcc3ec235
parent 297662 a2d21e14d40c4c28abd4e8e8bb588be66d0628bb
child 297664 15ae438a63dc4ec4d2725cb364356a786299f656
push id76851
push userahalberstadt@mozilla.com
push dateTue, 17 May 2016 13:11:25 +0000
treeherdermozilla-inbound@971d076cfe29 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssmacleod
bugs1270506
milestone49.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1270506 - [mozlint] Add python flake8 linter, r=smacleod For now, only the following two directories will be linted: python/mozlint tools/lint New directories can be added by adding them to the 'include' directive in tools/lint/flake8.lint. They all default to the configuration specified in topsrcdir/.flake8. Subdirectories can override this configuration by creating their own .flake8 file. MozReview-Commit-ID: Eag48Lnkp3l
.flake8
python/mozlint/mozlint/result.py
python/mozlint/mozlint/roller.py
python/mozlint/mozlint/types.py
python/mozlint/test/linters/regex.lint
python/mozlint/test/linters/string.lint
tools/lint/docs/conf.py
tools/lint/docs/index.rst
tools/lint/docs/linters/flake8.rst
tools/lint/flake8.lint
tools/lint/mach_commands.py
new file mode 100644
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 99
+filename = *.py, *.lint
--- a/python/mozlint/mozlint/result.py
+++ b/python/mozlint/mozlint/result.py
@@ -29,25 +29,25 @@ class ResultContainer(object):
         'column',
         'hint',
         'source',
         'level',
         'rule',
         'lineoffset',
     )
 
-    def __init__(self, linter, path, message, lineno, column=1, hint=None,
-                 source=None, level='error', rule=None, lineoffset=None):
+    def __init__(self, linter, path, message, lineno, column=None, hint=None,
+                 source=None, level=None, rule=None, lineoffset=None):
         self.path = path
         self.message = message
         self.lineno = lineno
-        self.column = column
+        self.column = column or 1
         self.hint = hint
         self.source = source
-        self.level = level
+        self.level = level or 'error'
         self.linter = linter
         self.rule = rule
         self.lineoffset = lineoffset
 
     def __repr__(self):
         s = dumps(self, cls=ResultEncoder, indent=2)
         return "ResultContainer({})".format(s)
 
--- a/python/mozlint/mozlint/roller.py
+++ b/python/mozlint/mozlint/roller.py
@@ -1,12 +1,14 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import unicode_literals
+
 import signal
 import traceback
 from collections import defaultdict
 from Queue import Empty
 from multiprocessing import (
     Manager,
     Pool,
     cpu_count,
@@ -30,16 +32,19 @@ def _run_linters(queue, paths, **lintarg
         # Ideally we would pass the entire LINTER definition as an argument
         # to the worker instead of re-parsing it. But passing a function from
         # a dynamically created module (with imp) does not seem to be possible
         # with multiprocessing on Windows.
         linter = parse(linter_path)
         func = supported_types[linter['type']]
         res = func(paths, linter, **lintargs) or []
 
+        if isinstance(res, basestring):
+            continue
+
         for r in res:
             results[r.path].append(r)
 
 
 def _run_worker(*args, **lintargs):
     try:
         return _run_linters(*args, **lintargs)
     except:
--- a/python/mozlint/mozlint/types.py
+++ b/python/mozlint/mozlint/types.py
@@ -25,16 +25,17 @@ class BaseType(object):
                          the definition, but passed in by a consumer.
         :returns: A list of :class:`~result.ResultContainer` objects.
         """
         exclude = lintargs.get('exclude', [])
         exclude.extend(linter.get('exclude', []))
 
         paths, exclude = filterpaths(paths, linter.get('include'), exclude)
         if not paths:
+            print("{}: No files to lint for specified paths!".format(linter['name']))
             return
 
         lintargs['exclude'] = exclude
         if self.batch:
             return self._lint(paths, linter, **lintargs)
 
         errors = []
         for p in paths:
--- a/python/mozlint/test/linters/regex.lint
+++ b/python/mozlint/test/linters/regex.lint
@@ -1,14 +1,15 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 
 LINTER = {
     'name': "RegexLinter",
-    'description': "Make sure the string 'foobar' never appears in a js variable files because it is bad.",
+    'description': "Make sure the string 'foobar' never appears "
+                   "in a js variable file because it is bad.",
     'rule': 'no-foobar',
     'include': [
         '**/*.js',
         '**/*.jsm',
     ],
     'type': 'regex',
     'payload': 'foobar',
 }
--- a/python/mozlint/test/linters/string.lint
+++ b/python/mozlint/test/linters/string.lint
@@ -1,14 +1,15 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 
 LINTER = {
     'name': "StringLinter",
-    'description': "Make sure the string 'foobar' never appears in browser js files because it is bad.",
+    'description': "Make sure the string 'foobar' never appears "
+                   "in browser js files because it is bad.",
     'rule': 'no-foobar',
     'include': [
         '**/*.js',
         '**/*.jsm',
     ],
     'type': 'string',
     'payload': 'foobar',
 }
--- a/tools/lint/docs/conf.py
+++ b/tools/lint/docs/conf.py
@@ -1,19 +1,17 @@
 # -*- coding: utf-8 -*-
 #
 # mozlint documentation build configuration file, created by
 # sphinx-quickstart on Fri Nov 27 17:38:49 2015.
 #
 # This file is execfile()d with the current directory set to its
 # containing dir.
 
-import sys
 import os
-import shlex
 
 # -- General configuration ------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
     'sphinx.ext.autodoc',
--- a/tools/lint/docs/index.rst
+++ b/tools/lint/docs/index.rst
@@ -21,15 +21,16 @@ 2. It provides a streamlined interface f
 like mach, mozreview and taskcluster.
 
 .. toctree::
   :caption: Linting User Guide
   :maxdepth: 2
 
   usage
   create
+  linters/flake8
 
 Indices and tables
 ==================
 
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`
new file mode 100644
--- /dev/null
+++ b/tools/lint/docs/linters/flake8.rst
@@ -0,0 +1,44 @@
+Flake8
+======
+
+`Flake8`_ is a popular lint wrapper for python. Under the hood, it runs three other tools and
+combines their results:
+
+* `pep8`_ for checking style
+* `pyflakes`_ for checking syntax
+* `mccabe`_ for checking complexity
+
+
+Run Locally
+-----------
+
+The mozlint integration of flake8 can be run using mach:
+
+.. parsed-literal::
+
+    $ mach lint --linter flake8 <file paths>
+
+Alternatively, omit the ``--linter flake8`` and run all configured linters, which will include
+flake8.
+
+
+Configuration
+-------------
+
+Only directories explicitly whitelisted will have flake8 run against them. To enable flake8 linting
+in a source directory, it must be added to the ``include`` directive in ```tools/lint/flake8.lint``.
+If you wish to exclude a subdirectory of an included one, you can add it to the ``exclude``
+directive.
+
+The default configuration file lives in ``topsrcdir/.flake8``. The default configuration can be
+overriden for a given subdirectory by creating a new ``.flake8`` file in the subdirectory. Be warned
+that ``.flake8`` files cannot inherit from one another, so all configuration you wish to keep must
+be re-defined.
+
+For an overview of the supported configuration, see `flake8's documentation`_.
+
+.. _Flake8: https://flake8.readthedocs.io/en/latest/
+.. _pep8: http://pep8.readthedocs.io/en/latest/
+.. _pyflakes: https://github.com/pyflakes/pyflakes
+.. _mccabe: https://github.com/pycqa/mccabe
+.. _flake8's documentation: https://flake8.readthedocs.io/en/latest/config.html
new file mode 100644
--- /dev/null
+++ b/tools/lint/flake8.lint
@@ -0,0 +1,107 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import subprocess
+
+from mozlint import result
+
+
+FLAKE8_NOT_FOUND = """
+Could not find flake8! Install flake8 and try again.
+
+    $ pip install flake8
+""".strip()
+
+
+LINE_OFFSETS = {
+    # continuation line under-indented for hanging indent
+    'E121': (-1, 2),
+    # continuation line missing indentation or outdented
+    'E122': (-1, 2),
+    # continuation line over-indented for hanging indent
+    'E126': (-1, 2),
+    # continuation line over-indented for visual indent
+    'E127': (-1, 2),
+    # continuation line under-indented for visual indent
+    'E128': (-1, 2),
+    # continuation line unaligned for hanging indend
+    'E131': (-1, 2),
+    # expected 1 blank line, found 0
+    'E301': (-1, 2),
+    # expected 2 blank lines, found 1
+    'E302': (-2, 3),
+}
+"""Maps a flake8 error to a lineoffset tuple.
+
+The offset is of the form (lineno_offset, num_lines) and is passed
+to the lineoffset property of `ResultContainer`.
+"""
+
+
+def lint(files, **lintargs):
+    import which
+
+    binary = os.environ.get('FLAKE8')
+    if not binary:
+        try:
+            binary = which.which('flake8')
+        except which.WhichError:
+            pass
+
+    if not binary:
+        print(FLAKE8_NOT_FOUND)
+        return 1
+
+    cmdargs = [
+        binary,
+        '--format', '{"path":"%(path)s","lineno":%(row)s,'
+                    '"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}',
+    ]
+
+    exclude = lintargs.get('exclude')
+    if exclude:
+        cmdargs += ['--exclude', ','.join(lintargs['exclude'])]
+
+    cmdargs += files
+
+    proc = subprocess.Popen(cmdargs, stdout=subprocess.PIPE, env=os.environ)
+    output = proc.communicate()[0]
+
+    if not output:
+        return []
+
+    results = []
+    for line in output.splitlines():
+        try:
+            res = json.loads(line)
+        except ValueError:
+            continue
+
+        if 'code' in res:
+            if res['code'].startswith('W'):
+                res['level'] = 'warning'
+
+            if res['code'] in LINE_OFFSETS:
+                res['lineoffset'] = LINE_OFFSETS[res['code']]
+
+        results.append(result.from_linter(LINTER, **res))
+
+    return results
+
+
+LINTER = {
+    'name': "flake8",
+    'description': "Python linter",
+    'include': [
+        'python/mozlint',
+        'tools/lint',
+    ],
+    'exclude': [],
+    'type': 'external',
+    'payload': lint,
+}
--- a/tools/lint/mach_commands.py
+++ b/tools/lint/mach_commands.py
@@ -1,15 +1,14 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import argparse
 import os
 
 from mozbuild.base import (
     MachCommandBase,
 )
 
 
 from mach.decorators import (
@@ -44,18 +43,18 @@ class MachCommands(MachCommandBase):
     def lint(self, paths, linters=None, fmt='stylish', **lintargs):
         """Run linters."""
         from mozlint import LintRoller, formatters
 
         paths = paths or ['.']
 
         lint_files = self.find_linters(linters)
 
-        lintargs['exclude'] = 'obj*'
-        lint = LintRoller(lintargs=lintargs)
+        lintargs['exclude'] = ['obj*']
+        lint = LintRoller(**lintargs)
         lint.read(lint_files)
 
         # run all linters
         results = lint.roll(paths)
 
         formatter = formatters.get(fmt)
         print(formatter(results))