Bug 1367092 - [flake8] Run flake8 programmatically instead of via a subprocess, r=egao
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Fri, 22 Feb 2019 21:14:06 +0000
changeset 518787 cd46126bc5924f041570d7adc889c394921930ba
parent 518786 ba172b704def575675d1303eef8f0b1c8ca33491
child 518788 41fdb372e22cf213eb76b966e357772f89a77710
push id10862
push userffxbld-merge
push dateMon, 11 Mar 2019 13:01:11 +0000
treeherdermozilla-beta@a2e7f5c935da [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersegao
bugs1367092
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1367092 - [flake8] Run flake8 programmatically instead of via a subprocess, r=egao This is required for a future commit which will monkeypatch flake8's configuration to fit our needs. But it has a couple nice benefits anyway: 1. Less process overhead. 2. Less complexity around handling SIGINT. 3. Less complexity in the code. Depends on D20494 Differential Revision: https://phabricator.services.mozilla.com/D20495
tools/lint/python/flake8.py
--- a/tools/lint/python/flake8.py
+++ b/tools/lint/python/flake8.py
@@ -1,25 +1,25 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import absolute_import, print_function
+
 import json
 import os
 import platform
-import signal
 import subprocess
 import sys
 
-from mozprocess import ProcessHandlerMixin
+import mozfile
 
 from mozlint import result
 from mozlint.util import pip
 
-
 here = os.path.abspath(os.path.dirname(__file__))
 FLAKE8_REQUIREMENTS_PATH = os.path.join(here, 'flake8_requirements.txt')
 
 FLAKE8_NOT_FOUND = """
 Could not find flake8! Install flake8 and try again.
 
     $ pip install -U --require-hashes -r {}
 """.strip().format(FLAKE8_REQUIREMENTS_PATH)
@@ -57,80 +57,64 @@ to the lineoffset property of an `Issue`
 
 # We use sys.prefix to find executables as that gets modified with
 # virtualenv's activate_this.py, whereas sys.executable doesn't.
 if platform.system() == 'Windows':
     bindir = os.path.join(sys.prefix, 'Scripts')
 else:
     bindir = os.path.join(sys.prefix, 'bin')
 
-results = []
+
+def setup(root):
+    if not pip.reinstall_program(FLAKE8_REQUIREMENTS_PATH):
+        print(FLAKE8_INSTALL_ERROR)
+        return 1
 
 
-class Flake8Process(ProcessHandlerMixin):
-    def __init__(self, config, *args, **kwargs):
-        self.config = config
-        kwargs['processOutputLine'] = [self.process_line]
-        ProcessHandlerMixin.__init__(self, *args, **kwargs)
+def lint(paths, config, **lintargs):
+    from flake8.main.application import Application
+
+    config_path = os.path.join(lintargs['root'], '.flake8')
+    exclude = config.get('exclude', [])
+
+    if lintargs.get('fix'):
+        fix_cmd = [
+            os.path.join(bindir, 'autopep8'),
+            '--global-config', config_path,
+            '--in-place', '--recursive',
+        ]
+
+        if exclude:
+            fix_cmd.extend(['--exclude', ','.join(exclude)])
 
-    def process_line(self, line):
+        subprocess.call(fix_cmd + paths)
+
+    output_file = mozfile.NamedTemporaryFile()
+    flake8_cmd = [
+        os.path.join(bindir, 'flake8'),
+        '--config', config_path,
+        '--output-file', output_file.name,
+        '--format', '{"path":"%(path)s","lineno":%(row)s,'
+                    '"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}',
+        '--filename', ','.join(['*.{}'.format(e) for e in config['extensions']]),
+    ] + paths
+
+    # Run flake8.
+    results = []
+    app = Application()
+    app.run(flake8_cmd)
+
+    def process_line(line):
         # Escape slashes otherwise JSON conversion will not work
         line = line.replace('\\', '\\\\')
         try:
             res = json.loads(line)
         except ValueError:
             print('Non JSON output from linter, will not be processed: {}'.format(line))
             return
 
         if res.get('code') in LINE_OFFSETS:
             res['lineoffset'] = LINE_OFFSETS[res['code']]
 
-        results.append(result.from_config(self.config, **res))
-
-    def run(self, *args, **kwargs):
-        # flake8 seems to handle SIGINT poorly. Handle it here instead
-        # so we can kill the process without a cryptic traceback.
-        orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
-        ProcessHandlerMixin.run(self, *args, **kwargs)
-        signal.signal(signal.SIGINT, orig)
-
-
-def setup(root):
-    if not pip.reinstall_program(FLAKE8_REQUIREMENTS_PATH):
-        print(FLAKE8_INSTALL_ERROR)
-        return 1
-
-
-def lint(paths, config, **lintargs):
-    # TODO don't store results in a global
-    global results
-    results = []
+        results.append(result.from_config(config, **res))
 
-    config_path = os.path.join(lintargs['root'], '.flake8')
-    cmdargs = [
-        os.path.join(bindir, 'flake8'),
-        '--config', config_path,
-        '--format', '{"path":"%(path)s","lineno":%(row)s,'
-                    '"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}',
-        '--filename', ','.join(['*.{}'.format(e) for e in config['extensions']]),
-    ]
-
-    if lintargs.get('fix'):
-        fix_cmdargs = [
-            os.path.join(bindir, 'autopep8'),
-            '--global-config', config_path,
-            '--in-place', '--recursive',
-        ]
-
-        if config.get('exclude'):
-            fix_cmdargs.extend(['--exclude', ','.join(config['exclude'])])
-
-        subprocess.call(fix_cmdargs + paths)
-
-    proc = Flake8Process(config, cmdargs + paths)
-    proc.run()
-    try:
-        proc.wait()
-    except KeyboardInterrupt:
-        proc.kill()
-        return 1
-
+    map(process_line, output_file.readlines())
     return results