Bug 1270506 - [mozlint] Add python flake8 linter, r?smacleod draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 05 May 2016 17:21:12 -0400
changeset 364414 654d411978a76776ee96e616c5f3369a026e0be4
parent 364413 558894c03922d3709c90c59d610485b6cd8beb5e
child 520273 b3e36234cf5318b5f9df8efe3b091c7bbe679444
push id17445
push userahalberstadt@mozilla.com
push dateFri, 06 May 2016 16:51:33 +0000
reviewerssmacleod
bugs1270506
milestone49.0a1
Bug 1270506 - [mozlint] Add python flake8 linter, r?smacleod For now, only the following two directories will be linted: python/mozlint tools/lint New directories can be added by adding them to the 'include' directive in tools/lint/flake8.lint. They all default to the configuration specified in topsrcdir/.flake8. Subdirectories can override this configuration by creating their own .flake8 file. MozReview-Commit-ID: Eag48Lnkp3l
.flake8
python/mozlint/mozlint/result.py
python/mozlint/mozlint/roller.py
python/mozlint/test/linters/regex.lint
python/mozlint/test/linters/string.lint
tools/lint/docs/conf.py
tools/lint/flake8.lint
tools/lint/mach_commands.py
new file mode 100644
--- /dev/null
+++ b/.flake8
@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 99
+filename = *.py, *.lint
--- a/python/mozlint/mozlint/result.py
+++ b/python/mozlint/mozlint/result.py
@@ -29,25 +29,25 @@ class ResultContainer(object):
         'column',
         'hint',
         'source',
         'level',
         'rule',
         'lineoffset',
     )
 
-    def __init__(self, linter, path, message, lineno, column=1, hint=None,
-                 source=None, level='error', rule=None, lineoffset=None):
+    def __init__(self, linter, path, message, lineno, column=None, hint=None,
+                 source=None, level=None, rule=None, lineoffset=None):
         self.path = path
         self.message = message
         self.lineno = lineno
-        self.column = column
+        self.column = column or 1
         self.hint = hint
         self.source = source
-        self.level = level
+        self.level = level or 'error'
         self.linter = linter
         self.rule = rule
         self.lineoffset = lineoffset
 
     def __repr__(self):
         s = dumps(self, cls=ResultEncoder, indent=2)
         return "ResultContainer({})".format(s)
 
--- a/python/mozlint/mozlint/roller.py
+++ b/python/mozlint/mozlint/roller.py
@@ -1,15 +1,17 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import unicode_literals
+
 import signal
 import traceback
-from collections import defaultdict
+from collections import defaultdict, Iterable
 from Queue import Empty
 from multiprocessing import (
     Manager,
     Pool,
     cpu_count,
 )
 
 from .errors import LintersNotConfigured
@@ -30,16 +32,19 @@ def _run_linters(queue, paths, **lintarg
         # Ideally we would pass the entire LINTER definition as an argument
         # to the worker instead of re-parsing it. But passing a function from
         # a dynamically created module (with imp) does not seem to be possible
         # with multiprocessing on Windows.
         linter = parse(linter_path)
         func = supported_types[linter['type']]
         res = func(paths, linter, **lintargs) or []
 
+        if not isinstance(res, Iterable):
+            continue
+
         for r in res:
             results[r.path].append(r)
 
 
 def _run_worker(*args, **lintargs):
     try:
         return _run_linters(*args, **lintargs)
     except:
--- a/python/mozlint/test/linters/regex.lint
+++ b/python/mozlint/test/linters/regex.lint
@@ -1,14 +1,15 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 
 LINTER = {
     'name': "RegexLinter",
-    'description': "Make sure the string 'foobar' never appears in a js variable files because it is bad.",
+    'description': "Make sure the string 'foobar' never appears "
+                   "in a js variable files because it is bad.",
     'rule': 'no-foobar',
     'include': [
         '**/*.js',
         '**/*.jsm',
     ],
     'type': 'regex',
     'payload': 'foobar',
 }
--- a/python/mozlint/test/linters/string.lint
+++ b/python/mozlint/test/linters/string.lint
@@ -1,14 +1,15 @@
 # -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
 # vim: set filetype=python:
 
 LINTER = {
     'name': "StringLinter",
-    'description': "Make sure the string 'foobar' never appears in browser js files because it is bad.",
+    'description': "Make sure the string 'foobar' never appears "
+                   "in browser js files because it is bad.",
     'rule': 'no-foobar',
     'include': [
         '**/*.js',
         '**/*.jsm',
     ],
     'type': 'string',
     'payload': 'foobar',
 }
--- a/tools/lint/docs/conf.py
+++ b/tools/lint/docs/conf.py
@@ -1,19 +1,17 @@
 # -*- coding: utf-8 -*-
 #
 # mozlint documentation build configuration file, created by
 # sphinx-quickstart on Fri Nov 27 17:38:49 2015.
 #
 # This file is execfile()d with the current directory set to its
 # containing dir.
 
-import sys
 import os
-import shlex
 
 # -- General configuration ------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
     'sphinx.ext.autodoc',
new file mode 100644
--- /dev/null
+++ b/tools/lint/flake8.lint
@@ -0,0 +1,77 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import subprocess
+
+from mozlint import result
+
+
+FLAKE8_NOT_FOUND = """
+Could not find flake8! Install flake8 and try again.
+
+    $ pip install flake8
+""".strip()
+
+
+def lint(files, **lintargs):
+    import which
+
+    binary = os.environ.get('FLAKE8')
+    if not binary:
+        try:
+            binary = which.which('flake8')
+        except which.WhichError:
+            pass
+
+    if not binary:
+        print(FLAKE8_NOT_FOUND)
+        return 1
+
+    cmdargs = [
+        binary,
+        '--format', '{"path":"%(path)s","lineno":%(row)s,'
+                    '"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}',
+    ]
+
+    exclude = lintargs.get('exclude')
+    if exclude:
+        cmdargs += ['--exclude', ','.join(lintargs['exclude'])]
+
+    cmdargs += files
+
+    proc = subprocess.Popen(cmdargs, stdout=subprocess.PIPE, env=os.environ)
+    output = proc.communicate()[0]
+
+    if not output:
+        return []
+
+    results = []
+    for line in output.splitlines():
+        try:
+            res = json.loads(line)
+        except ValueError:
+            continue
+
+        if 'code' in res and res['code'].startswith('W'):
+            res['level'] = 'warning'
+        results.append(result.from_linter(LINTER, **res))
+
+    return results
+
+
+LINTER = {
+    'name': "flake8",
+    'description': "Python linter",
+    'include': [
+        'python/mozlint',
+        'tools/lint',
+    ],
+    'exclude': [],
+    'type': 'external',
+    'payload': lint,
+}
--- a/tools/lint/mach_commands.py
+++ b/tools/lint/mach_commands.py
@@ -1,15 +1,14 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import argparse
 import os
 
 from mozbuild.base import (
     MachCommandBase,
 )
 
 
 from mach.decorators import (
@@ -44,18 +43,18 @@ class MachCommands(MachCommandBase):
     def lint(self, paths, linters=None, fmt='stylish', **lintargs):
         """Run linters."""
         from mozlint import LintRoller, formatters
 
         paths = paths or ['.']
 
         lint_files = self.find_linters(linters)
 
-        lintargs['exclude'] = 'obj*'
-        lint = LintRoller(lintargs=lintargs)
+        lintargs['exclude'] = ['obj*']
+        lint = LintRoller(**lintargs)
         lint.read(lint_files)
 
         # run all linters
         results = lint.roll(paths)
 
         formatter = formatters.get(fmt)
         print(formatter(results))