Bug 1288432 - Use new mozlint configuration for flake8 linter, r?bc draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Fri, 02 Jun 2017 09:46:01 -0400
changeset 588434 c7443a152ffa8f6837ed55553e633f123c78a3cc
parent 588433 d6b71a194cef1685f27f9ef5f151ae9acdde498e
child 588435 a518dafb94767020e8f670f7b4e71cbf22af1cc7
push id62032
push userahalberstadt@mozilla.com
push dateFri, 02 Jun 2017 19:52:29 +0000
reviewersbc
bugs1288432
milestone55.0a1
Bug 1288432 - Use new mozlint configuration for flake8 linter, r?bc MozReview-Commit-ID: 9vC6lI2j5nS
.flake8
taskcluster/docker/lint/Dockerfile
tools/lint/flake8.lint.py
tools/lint/flake8.yml
tools/lint/flake8/flake8_requirements.txt
tools/lint/flake8_/__init__.py
tools/lint/flake8_/flake8_requirements.txt
--- a/.flake8
+++ b/.flake8
@@ -1,5 +1,4 @@
 [flake8]
 # See http://pep8.readthedocs.io/en/latest/intro.html#configuration
 ignore = E121, E123, E126, E129, E133, E226, E241, E242, E704, W503, E402
 max-line-length = 99
-filename = *.py, +.lint
--- a/taskcluster/docker/lint/Dockerfile
+++ b/taskcluster/docker/lint/Dockerfile
@@ -12,18 +12,18 @@ ADD topsrcdir/python/mozbuild/mozbuild/a
 ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
 
 # %include taskcluster/docker/recipes/install-node.sh
 ADD topsrcdir/taskcluster/docker/recipes/install-node.sh /build/install-node.sh
 
 # %include taskcluster/docker/recipes/install-mercurial.sh
 ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /build/install-mercurial.sh
 ADD system-setup.sh /tmp/system-setup.sh
-# %include tools/lint/flake8/flake8_requirements.txt
-ADD topsrcdir/tools/lint/flake8/flake8_requirements.txt /tmp/flake8_requirements.txt
+# %include tools/lint/flake8_/flake8_requirements.txt
+ADD topsrcdir/tools/lint/flake8_/flake8_requirements.txt /tmp/flake8_requirements.txt
 # %include tools/lint/tox/tox_requirements.txt
 ADD topsrcdir/tools/lint/tox/tox_requirements.txt /tmp/tox_requirements.txt
 RUN bash /tmp/system-setup.sh
 
 # %include taskcluster/docker/recipes/run-task
 ADD topsrcdir/taskcluster/docker/recipes/run-task /home/worker/bin/run-task
 RUN chown -R worker:worker /home/worker/bin && chmod 755 /home/worker/bin/*
 
new file mode 100644
--- /dev/null
+++ b/tools/lint/flake8.yml
@@ -0,0 +1,23 @@
+flake8:
+    description: Python linter
+    include:
+        - layout/tools/reftest
+        - python/mozlint
+        - security/manager
+        - taskcluster
+        - testing/firefox-ui
+        - testing/marionette/client
+        - testing/marionette/harness
+        - testing/marionette/puppeteer
+        - testing/mozbase
+        - testing/mochitest
+        - testing/talos/
+        - tools/git
+        - tools/lint
+        - tools/mercurial
+        - toolkit/components/telemetry
+    exclude:
+        - testing/mochitest/pywebsocket
+    extensions: ['.py']
+    type: external
+    payload: flake8_:lint
rename from tools/lint/flake8.lint.py
rename to tools/lint/flake8_/__init__.py
--- a/tools/lint/flake8.lint.py
+++ b/tools/lint/flake8_/__init__.py
@@ -1,27 +1,25 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
 import os
 import signal
 import subprocess
 
 import which
-from mozprocess import ProcessHandler
+from mozprocess import ProcessHandlerMixin
 
 from mozlint import result
 
 
 here = os.path.abspath(os.path.dirname(__file__))
-FLAKE8_REQUIREMENTS_PATH = os.path.join(here, 'flake8', 'flake8_requirements.txt')
+FLAKE8_REQUIREMENTS_PATH = os.path.join(here, 'flake8_requirements.txt')
 
 FLAKE8_NOT_FOUND = """
 Could not find flake8! Install flake8 and try again.
 
     $ pip install -U --require-hashes -r {}
 """.strip().format(FLAKE8_REQUIREMENTS_PATH)
 
 
@@ -50,52 +48,49 @@ LINE_OFFSETS = {
     'E302': (-2, 3),
 }
 """Maps a flake8 error to a lineoffset tuple.
 
 The offset is of the form (lineno_offset, num_lines) and is passed
 to the lineoffset property of `ResultContainer`.
 """
 
-EXTENSIONS = ['.py', '.lint']
 results = []
 
 
-def process_line(line):
-    # Escape slashes otherwise JSON conversion will not work
-    line = line.replace('\\', '\\\\')
-    try:
-        res = json.loads(line)
-    except ValueError:
-        print('Non JSON output from linter, will not be processed: {}'.format(line))
-        return
+class Flake8Process(ProcessHandlerMixin):
+    def __init__(self, config, *args, **kwargs):
+        self.config = config
+        kwargs['processOutputLine'] = [self.process_line]
+        ProcessHandlerMixin.__init__(self, *args, **kwargs)
 
-    if 'code' in res:
-        if res['code'].startswith('W'):
-            res['level'] = 'warning'
-
-        if res['code'] in LINE_OFFSETS:
-            res['lineoffset'] = LINE_OFFSETS[res['code']]
+    def process_line(self, line):
+        # Escape slashes otherwise JSON conversion will not work
+        line = line.replace('\\', '\\\\')
+        try:
+            res = json.loads(line)
+        except ValueError:
+            print('Non JSON output from linter, will not be processed: {}'.format(line))
+            return
 
-    results.append(result.from_linter(LINTER, **res))
+        if 'code' in res:
+            if res['code'].startswith('W'):
+                res['level'] = 'warning'
 
+            if res['code'] in LINE_OFFSETS:
+                res['lineoffset'] = LINE_OFFSETS[res['code']]
 
-def run_process(cmdargs):
-    # flake8 seems to handle SIGINT poorly. Handle it here instead
-    # so we can kill the process without a cryptic traceback.
-    orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
-    proc = ProcessHandler(cmdargs, env=os.environ,
-                          processOutputLine=process_line)
-    proc.run()
-    signal.signal(signal.SIGINT, orig)
+        results.append(result.from_config(self.config, **res))
 
-    try:
-        proc.wait()
-    except KeyboardInterrupt:
-        proc.kill()
+    def run(self, *args, **kwargs):
+        # flake8 seems to handle SIGINT poorly. Handle it here instead
+        # so we can kill the process without a cryptic traceback.
+        orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
+        ProcessHandlerMixin.run(self, *args, **kwargs)
+        signal.signal(signal.SIGINT, orig)
 
 
 def get_flake8_binary():
     """
     Returns the path of the first flake8 binary available
     if not found returns None
     """
     binary = os.environ.get('FLAKE8')
@@ -129,17 +124,26 @@ def reinstall_flake8():
     if _run_pip('install', '-U',
                 '--require-hashes', '-r',
                 FLAKE8_REQUIREMENTS_PATH):
         return True
 
     return False
 
 
-def lint(files, **lintargs):
+def run_process(config, cmd):
+    proc = Flake8Process(config, cmd)
+    proc.run()
+    try:
+        proc.wait()
+    except KeyboardInterrupt:
+        proc.kill()
+
+
+def lint(files, config, **lintargs):
 
     if not reinstall_flake8():
         print(FLAKE8_INSTALL_ERROR)
         return 1
 
     binary = get_flake8_binary()
 
     cmdargs = [
@@ -152,48 +156,21 @@ def lint(files, **lintargs):
     # it gets picked up. This means only .flake8 files that live in
     # directories that are explicitly included will be considered.
     # See bug 1277851
     no_config = []
     for f in files:
         if not os.path.isfile(os.path.join(f, '.flake8')):
             no_config.append(f)
             continue
-        run_process(cmdargs+[f])
+        run_process(config, cmdargs+[f])
 
     # XXX For some reason passing in --exclude results in flake8 not using
     # the local .flake8 file. So for now only pass in --exclude if there
     # is no local config.
     exclude = lintargs.get('exclude')
     if exclude:
         cmdargs += ['--exclude', ','.join(lintargs['exclude'])]
 
     if no_config:
-        run_process(cmdargs+no_config)
+        run_process(config, cmdargs+no_config)
 
     return results
-
-
-LINTER = {
-    'name': "flake8",
-    'description': "Python linter",
-    'include': [
-        'layout/tools/reftest',
-        'python/mozlint',
-        'security/manager',
-        'taskcluster',
-        'testing/firefox-ui',
-        'testing/marionette/client',
-        'testing/marionette/harness',
-        'testing/marionette/puppeteer',
-        'testing/mozbase',
-        'testing/mochitest',
-        'testing/talos/',
-        'tools/git',
-        'tools/lint',
-        'tools/mercurial',
-        'toolkit/components/telemetry',
-    ],
-    'exclude': ['testing/mochitest/pywebsocket'],
-    'extensions': EXTENSIONS,
-    'type': 'external',
-    'payload': lint,
-}
rename from tools/lint/flake8/flake8_requirements.txt
rename to tools/lint/flake8_/flake8_requirements.txt