Bug 1436639 - [lint] Make sure flake8 is run with same python as |mach lint| was, r=jmaher
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 07 Feb 2018 23:28:33 -0500
changeset 456054 365a91ab41e21db14bed86a705536bd222bccfcd
parent 456053 21e0ed25f05a5f7c7b8213016511d0577708efbe
child 456055 dfcfef9a322db951210fbf7f86d77badb489b8ca
push id8799
push usermtabara@mozilla.com
push dateThu, 01 Mar 2018 16:46:23 +0000
treeherdermozilla-beta@15334014dc67 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1436639
milestone60.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1436639 - [lint] Make sure flake8 is run with same python as |mach lint| was, r=jmaher This fixes a bug which can happen when the default version of python differs from the version of python used with mach. For example, mach explicitly looks for python2.7. This means running |mach lint -l flake8| should also run flake8 with version 2.7. But if the default is python3, and flake8 is also installed there, the subprocess call that invokes flake8 will run under python3. This can lead to errors like "undefined name 'basestring'" and other 2to3 gotchas. This patch ensures that we run the flake8 binary (and the pip for installing flake8) from the same interpreter as mach, no matter the system default. MozReview-Commit-ID: HSuMzDsAvsW
tools/lint/python/flake8.py
tools/lint/python/flake8_requirements.txt
--- a/tools/lint/python/flake8.py
+++ b/tools/lint/python/flake8.py
@@ -1,19 +1,20 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
 import os
+import platform
 import signal
 import subprocess
+import sys
 from collections import defaultdict
 
-import which
 from mozprocess import ProcessHandlerMixin
 
 from mozlint import result
 from mozlint.pathutils import get_ancestors_by_name
 
 
 here = os.path.abspath(os.path.dirname(__file__))
 FLAKE8_REQUIREMENTS_PATH = os.path.join(here, 'flake8_requirements.txt')
@@ -50,16 +51,23 @@ LINE_OFFSETS = {
     'E302': (-2, 3),
 }
 """Maps a flake8 error to a lineoffset tuple.
 
 The offset is of the form (lineno_offset, num_lines) and is passed
 to the lineoffset property of `ResultContainer`.
 """
 
+# We use sys.prefix to find executables as that gets modified with
+# virtualenv's activate_this.py, whereas sys.executable doesn't.
+if platform.system() == 'Windows':
+    bindir = os.path.join(sys.prefix, 'Scripts')
+else:
+    bindir = os.path.join(sys.prefix, 'bin')
+
 results = []
 
 
 class Flake8Process(ProcessHandlerMixin):
     def __init__(self, config, *args, **kwargs):
         self.config = config
         kwargs['processOutputLine'] = [self.process_line]
         ProcessHandlerMixin.__init__(self, *args, **kwargs)
@@ -85,33 +93,22 @@ class Flake8Process(ProcessHandlerMixin)
     def run(self, *args, **kwargs):
         # flake8 seems to handle SIGINT poorly. Handle it here instead
         # so we can kill the process without a cryptic traceback.
         orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
         ProcessHandlerMixin.run(self, *args, **kwargs)
         signal.signal(signal.SIGINT, orig)
 
 
-def get_flake8_binary():
-    """
-    Returns the path of the first flake8 binary available
-    if not found returns None
-    """
-    try:
-        return which.which('flake8')
-    except which.WhichError:
-        return None
-
-
 def _run_pip(*args):
     """
     Helper function that runs pip with subprocess
     """
     try:
-        subprocess.check_output(['pip'] + list(args),
+        subprocess.check_output([os.path.join(bindir, 'pip')] + list(args),
                                 stderr=subprocess.STDOUT)
         return True
     except subprocess.CalledProcessError as e:
         print(e.output)
         return False
 
 
 def reinstall_flake8():
@@ -138,19 +135,18 @@ def run_process(config, cmd):
 
 def setup(root):
     if not reinstall_flake8():
         print(FLAKE8_INSTALL_ERROR)
         return 1
 
 
 def lint(paths, config, **lintargs):
-    binary = get_flake8_binary()
     cmdargs = [
-        binary,
+        os.path.join(bindir, 'flake8'),
         '--format', '{"path":"%(path)s","lineno":%(row)s,'
                     '"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}',
     ]
 
     # Run any paths with a .flake8 file in the directory separately so
     # it gets picked up. This means only .flake8 files that live in
     # directories that are explicitly included will be considered.
     # See bug 1277851
--- a/tools/lint/python/flake8_requirements.txt
+++ b/tools/lint/python/flake8_requirements.txt
@@ -12,8 +12,11 @@ pycodestyle==2.3.1 \
     --hash=sha256:682256a5b318149ca0d2a9185d365d8864a768a28db66a84a2ea946bcc426766
 enum34==1.1.6 \
     --hash=sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79 \
     --hash=sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a \
     --hash=sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1 \
     --hash=sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850
 configparser==3.5.0 \
     --hash=sha256:5308b47021bc2340965c371f0f058cc6971a04502638d4244225c49d80db273a
+setuptools==38.5.1 \
+    --hash=sha256:7ffe771abfae419fd104f93400b61c935b5af10bfe4dfeec7a1bd495694eea35 \
+    --hash=sha256:6425484c08e99a98a42209c25c3d325f749230b55284d66192784f941a7e6628