Bug 1288432 - Use new mozlint configuration for flake8 linter, r=bc
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Fri, 02 Jun 2017 09:46:01 -0400
changeset 410935 97d503af98d9ac9e6fdba3d8c7f2ffc6c74bac27
parent 410934 a9e96c22024990f36321b04be3a67796d98d020c
child 410936 0eb21ab64fcd026cc753fc6d9457499fab9045d5
push id7391
push usermtabara@mozilla.com
push dateMon, 12 Jun 2017 13:08:53 +0000
treeherdermozilla-beta@2191d7f87e2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbc
bugs1288432
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1288432 - Use new mozlint configuration for flake8 linter, r=bc MozReview-Commit-ID: 9vC6lI2j5nS
.flake8
taskcluster/docker/lint/Dockerfile
tools/lint/flake8.lint.py
tools/lint/flake8.yml
tools/lint/flake8/flake8_requirements.txt
tools/lint/flake8_/__init__.py
tools/lint/flake8_/flake8_requirements.txt
--- a/.flake8
+++ b/.flake8
@@ -1,5 +1,4 @@
 [flake8]
 # See http://pep8.readthedocs.io/en/latest/intro.html#configuration
 ignore = E121, E123, E126, E129, E133, E226, E241, E242, E704, W503, E402
 max-line-length = 99
-filename = *.py, +.lint
--- a/taskcluster/docker/lint/Dockerfile
+++ b/taskcluster/docker/lint/Dockerfile
@@ -12,18 +12,18 @@ ADD topsrcdir/python/mozbuild/mozbuild/a
 ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
 
 # %include taskcluster/docker/recipes/install-node.sh
 ADD topsrcdir/taskcluster/docker/recipes/install-node.sh /build/install-node.sh
 
 # %include taskcluster/docker/recipes/install-mercurial.sh
 ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /build/install-mercurial.sh
 ADD system-setup.sh /tmp/system-setup.sh
-# %include tools/lint/flake8/flake8_requirements.txt
-ADD topsrcdir/tools/lint/flake8/flake8_requirements.txt /tmp/flake8_requirements.txt
+# %include tools/lint/flake8_/flake8_requirements.txt
+ADD topsrcdir/tools/lint/flake8_/flake8_requirements.txt /tmp/flake8_requirements.txt
 # %include tools/lint/tox/tox_requirements.txt
 ADD topsrcdir/tools/lint/tox/tox_requirements.txt /tmp/tox_requirements.txt
 RUN bash /tmp/system-setup.sh
 
 # %include taskcluster/docker/recipes/run-task
 ADD topsrcdir/taskcluster/docker/recipes/run-task /home/worker/bin/run-task
 RUN chown -R worker:worker /home/worker/bin && chmod 755 /home/worker/bin/*
 
new file mode 100644
--- /dev/null
+++ b/tools/lint/flake8.yml
@@ -0,0 +1,23 @@
+flake8:
+    description: Python linter
+    include:
+        - layout/tools/reftest
+        - python/mozlint
+        - security/manager
+        - taskcluster
+        - testing/firefox-ui
+        - testing/marionette/client
+        - testing/marionette/harness
+        - testing/marionette/puppeteer
+        - testing/mozbase
+        - testing/mochitest
+        - testing/talos/
+        - tools/git
+        - tools/lint
+        - tools/mercurial
+        - toolkit/components/telemetry
+    exclude:
+        - testing/mochitest/pywebsocket
+    extensions: ['.py']
+    type: external
+    payload: flake8_:lint
rename from tools/lint/flake8.lint.py
rename to tools/lint/flake8_/__init__.py
--- a/tools/lint/flake8.lint.py
+++ b/tools/lint/flake8_/__init__.py
@@ -1,27 +1,25 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
 import os
 import signal
 import subprocess
 
 import which
-from mozprocess import ProcessHandler
+from mozprocess import ProcessHandlerMixin
 
 from mozlint import result
 
 
 here = os.path.abspath(os.path.dirname(__file__))
-FLAKE8_REQUIREMENTS_PATH = os.path.join(here, 'flake8', 'flake8_requirements.txt')
+FLAKE8_REQUIREMENTS_PATH = os.path.join(here, 'flake8_requirements.txt')
 
 FLAKE8_NOT_FOUND = """
 Could not find flake8! Install flake8 and try again.
 
     $ pip install -U --require-hashes -r {}
 """.strip().format(FLAKE8_REQUIREMENTS_PATH)
 
 
@@ -50,52 +48,49 @@ LINE_OFFSETS = {
     'E302': (-2, 3),
 }
 """Maps a flake8 error to a lineoffset tuple.
 
 The offset is of the form (lineno_offset, num_lines) and is passed
 to the lineoffset property of `ResultContainer`.
 """
 
-EXTENSIONS = ['.py', '.lint']
 results = []
 
 
-def process_line(line):
-    # Escape slashes otherwise JSON conversion will not work
-    line = line.replace('\\', '\\\\')
-    try:
-        res = json.loads(line)
-    except ValueError:
-        print('Non JSON output from linter, will not be processed: {}'.format(line))
-        return
+class Flake8Process(ProcessHandlerMixin):
+    def __init__(self, config, *args, **kwargs):
+        self.config = config
+        kwargs['processOutputLine'] = [self.process_line]
+        ProcessHandlerMixin.__init__(self, *args, **kwargs)
 
-    if 'code' in res:
-        if res['code'].startswith('W'):
-            res['level'] = 'warning'
-
-        if res['code'] in LINE_OFFSETS:
-            res['lineoffset'] = LINE_OFFSETS[res['code']]
+    def process_line(self, line):
+        # Escape slashes otherwise JSON conversion will not work
+        line = line.replace('\\', '\\\\')
+        try:
+            res = json.loads(line)
+        except ValueError:
+            print('Non JSON output from linter, will not be processed: {}'.format(line))
+            return
 
-    results.append(result.from_linter(LINTER, **res))
+        if 'code' in res:
+            if res['code'].startswith('W'):
+                res['level'] = 'warning'
 
+            if res['code'] in LINE_OFFSETS:
+                res['lineoffset'] = LINE_OFFSETS[res['code']]
 
-def run_process(cmdargs):
-    # flake8 seems to handle SIGINT poorly. Handle it here instead
-    # so we can kill the process without a cryptic traceback.
-    orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
-    proc = ProcessHandler(cmdargs, env=os.environ,
-                          processOutputLine=process_line)
-    proc.run()
-    signal.signal(signal.SIGINT, orig)
+        results.append(result.from_config(self.config, **res))
 
-    try:
-        proc.wait()
-    except KeyboardInterrupt:
-        proc.kill()
+    def run(self, *args, **kwargs):
+        # flake8 seems to handle SIGINT poorly. Handle it here instead
+        # so we can kill the process without a cryptic traceback.
+        orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
+        ProcessHandlerMixin.run(self, *args, **kwargs)
+        signal.signal(signal.SIGINT, orig)
 
 
 def get_flake8_binary():
     """
     Returns the path of the first flake8 binary available
     if not found returns None
     """
     binary = os.environ.get('FLAKE8')
@@ -129,17 +124,26 @@ def reinstall_flake8():
     if _run_pip('install', '-U',
                 '--require-hashes', '-r',
                 FLAKE8_REQUIREMENTS_PATH):
         return True
 
     return False
 
 
-def lint(files, **lintargs):
+def run_process(config, cmd):
+    proc = Flake8Process(config, cmd)
+    proc.run()
+    try:
+        proc.wait()
+    except KeyboardInterrupt:
+        proc.kill()
+
+
+def lint(files, config, **lintargs):
 
     if not reinstall_flake8():
         print(FLAKE8_INSTALL_ERROR)
         return 1
 
     binary = get_flake8_binary()
 
     cmdargs = [
@@ -152,48 +156,21 @@ def lint(files, **lintargs):
     # it gets picked up. This means only .flake8 files that live in
     # directories that are explicitly included will be considered.
     # See bug 1277851
     no_config = []
     for f in files:
         if not os.path.isfile(os.path.join(f, '.flake8')):
             no_config.append(f)
             continue
-        run_process(cmdargs+[f])
+        run_process(config, cmdargs+[f])
 
     # XXX For some reason passing in --exclude results in flake8 not using
     # the local .flake8 file. So for now only pass in --exclude if there
     # is no local config.
     exclude = lintargs.get('exclude')
     if exclude:
         cmdargs += ['--exclude', ','.join(lintargs['exclude'])]
 
     if no_config:
-        run_process(cmdargs+no_config)
+        run_process(config, cmdargs+no_config)
 
     return results
-
-
-LINTER = {
-    'name': "flake8",
-    'description': "Python linter",
-    'include': [
-        'layout/tools/reftest',
-        'python/mozlint',
-        'security/manager',
-        'taskcluster',
-        'testing/firefox-ui',
-        'testing/marionette/client',
-        'testing/marionette/harness',
-        'testing/marionette/puppeteer',
-        'testing/mozbase',
-        'testing/mochitest',
-        'testing/talos/',
-        'tools/git',
-        'tools/lint',
-        'tools/mercurial',
-        'toolkit/components/telemetry',
-    ],
-    'exclude': ['testing/mochitest/pywebsocket'],
-    'extensions': EXTENSIONS,
-    'type': 'external',
-    'payload': lint,
-}
rename from tools/lint/flake8/flake8_requirements.txt
rename to tools/lint/flake8_/flake8_requirements.txt