Bug 1285555 - [flake8] Automatically install/update flake8 package to correct version r?ahal draft
authorFrancesco Pischedda <francesco.pischedda@gmail.com>
Sat, 15 Oct 2016 09:45:22 +0200
changeset 428740 db7cd1b378b56e9036f041446addddcd5af1bad7
parent 425306 cb2dd5a34dd7b374500fedd72fe19df13c9a7a4d
child 534826 ccf955e5950464f09140be6dea6c13aad66c8490
push id33408
push userbmo:francesco.pischedda@gmail.com
push dateMon, 24 Oct 2016 15:39:26 +0000
reviewersahal
bugs1285555
milestone52.0a1
Bug 1285555 - [flake8] Automatically install/update flake8 package to correct version r?ahal Changed flake8 installation strategy, try to update it using testings/lint/flake8_requirements.txt moved flake8_requirements to tools/lint/flake8 directory extracted common pip invocations to _run_pip function minor fixes fixed calls to check_process (concatenating list to tuple) removed update_pip function and its error message variable if flake8 installation reports and error print pip command output MozReview-Commit-ID: Fmq22aL8dMq
testing/docker/lint/Dockerfile
testing/docker/lint/system-setup.sh
tools/lint/flake8.lint
tools/lint/flake8/flake8_requirements.txt
tools/lint/mach_commands.py
--- a/testing/docker/lint/Dockerfile
+++ b/testing/docker/lint/Dockerfile
@@ -9,16 +9,18 @@ RUN mkdir /build
 ADD topsrcdir/testing/docker/recipes/tooltool.py /build/tooltool.py
 
 # %include testing/mozharness/external_tools/robustcheckout.py
 ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
 
 # %include testing/docker/recipes/install-mercurial.sh
 ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /build/install-mercurial.sh
 ADD system-setup.sh /tmp/system-setup.sh
+# %include tools/lint/flake8/flake8_requirements.txt
+ADD topsrcdir/tools/lint/flake8/flake8_requirements.txt /tmp/flake8_requirements.txt
 RUN bash /tmp/system-setup.sh
 
 # %include testing/docker/recipes/run-task
 ADD topsrcdir/testing/docker/recipes/run-task /home/worker/bin/run-task
 RUN chown -R worker:worker /home/worker/bin && chmod 755 /home/worker/bin/*
 
 # Set variable normally configured at login, by the shells parent process, these
 # are taken from GNU su manual
--- a/testing/docker/lint/system-setup.sh
+++ b/testing/docker/lint/system-setup.sh
@@ -56,24 +56,13 @@ tar -C /usr/local --strip-components 1 -
 node -v  # verify
 npm -v
 
 ###
 # Flake8 Setup
 ###
 
 cd /setup
-cat >requirements.txt <<'EOF'
-flake8==2.5.4 \
-  --hash=sha256:fb5a67af4024622287a76abf6b7fe4fb3cfacf765a790976ce64f52c44c88e4a
-mccabe==0.4.0 \
-  --hash=sha256:cbc2938f6c01061bc6d21d0c838c2489664755cb18676f0734d7617f4577d09e
-pep8==1.7.0 \
-  --hash=sha256:4fc2e478addcf17016657dff30b2d8d611e8341fac19ccf2768802f6635d7b8a
-pyflakes==1.2.3 \
-  --hash=sha256:e87bac26c62ea5b45067cc89e4a12f56e1483f1f2cda17e7c9b375b9fd2f40da
-EOF
 
-pip install --require-hashes -r requirements.txt
+pip install --require-hashes -r /tmp/flake8_requirements.txt
 
 cd /
 rm -rf /setup
-
--- a/tools/lint/flake8.lint
+++ b/tools/lint/flake8.lint
@@ -2,30 +2,40 @@
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
 import os
 import signal
+import subprocess
 
 import which
 from mozprocess import ProcessHandler
 
 from mozlint import result
 
 
 FLAKE8_NOT_FOUND = """
 Could not find flake8! Install flake8 and try again.
 
-    $ pip install flake8
+    $ pip install -U --require-hashes -r flake8_requirements.txt
 """.strip()
 
 
+FLAKE8_INSTALL_ERROR = """
+Unable to install correct version of flake8
+Try to install it manually with:
+    $ pip install -U --require-hashes -r flake8/flake8_requirements.txt
+""".strip()
+
+here = os.path.abspath(os.path.dirname(__file__))
+FLAKE8_REQUIREMENTS_PATH = os.path.join(here, 'flake8', 'flake8_requirements.txt')
+
 LINE_OFFSETS = {
     # continuation line under-indented for hanging indent
     'E121': (-1, 2),
     # continuation line missing indentation or outdented
     'E122': (-1, 2),
     # continuation line over-indented for hanging indent
     'E126': (-1, 2),
     # continuation line over-indented for visual indent
@@ -78,27 +88,63 @@ def run_process(cmdargs):
     signal.signal(signal.SIGINT, orig)
 
     try:
         proc.wait()
     except KeyboardInterrupt:
         proc.kill()
 
 
-def lint(files, **lintargs):
+def get_flake8_binary():
+    """
+    Returns the path of the first flake8 binary available
+    if not found returns None
+    """
     binary = os.environ.get('FLAKE8')
-    if not binary:
-        try:
-            binary = which.which('flake8')
-        except which.WhichError:
-            pass
+    if binary:
+        return binary
+
+    try:
+        return which.which('flake8')
+    except which.WhichError:
+        return None
+
+
+def _run_pip(*args):
+    """
+    Helper function that runs pip with subprocess
+    """
+    try:
+        out = subprocess.check_output(['pip'] + list(args))
+        return True
+    except subprocess.CalledProcessError as e:
+        print(out)
+        return False
+
 
-    if not binary:
-        print(FLAKE8_NOT_FOUND)
-        return []
+def reinstall_flake8():
+    """
+    Try to install flake8 at the target version, returns True on success
+    otherwise prints the otuput of the pip command and returns False
+    """
+    if _run_pip('install', '-U',
+                '--require-hashes', '-r',
+                FLAKE8_REQUIREMENTS_PATH):
+        return True
+    else:
+        return False
+
+
+def lint(files, **lintargs):
+
+    if not reinstall_flake8():
+        print(FLAKE8_INSTALL_ERROR)
+        return 1
+
+    binary = get_flake8_binary()
 
     cmdargs = [
         binary,
         '--format', '{"path":"%(path)s","lineno":%(row)s,'
                     '"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}',
     ]
 
     # Run any paths with a .flake8 file in the directory separately so
@@ -114,17 +160,16 @@ def lint(files, **lintargs):
 
     # XXX For some reason passing in --exclude results in flake8 not using
     # the local .flake8 file. So for now only pass in --exclude if there
     # is no local config.
     exclude = lintargs.get('exclude')
     if exclude:
         cmdargs += ['--exclude', ','.join(lintargs['exclude'])]
 
-
     if no_config:
         run_process(cmdargs+no_config)
 
     return results
 
 
 LINTER = {
     'name': "flake8",
new file mode 100644
--- /dev/null
+++ b/tools/lint/flake8/flake8_requirements.txt
@@ -0,0 +1,4 @@
+flake8==2.5.4 --hash=sha256:fb5a67af4024622287a76abf6b7fe4fb3cfacf765a790976ce64f52c44c88e4a
+mccabe==0.4.0 --hash=sha256:cbc2938f6c01061bc6d21d0c838c2489664755cb18676f0734d7617f4577d09e
+pep8==1.7.0 --hash=sha256:4fc2e478addcf17016657dff30b2d8d611e8341fac19ccf2768802f6635d7b8a
+pyflakes==1.2.3 --hash=sha256:e87bac26c62ea5b45067cc89e4a12f56e1483f1f2cda17e7c9b375b9fd2f40da
--- a/tools/lint/mach_commands.py
+++ b/tools/lint/mach_commands.py
@@ -34,16 +34,17 @@ class MachCommands(MachCommandBase):
         'lint', category='devenv',
         description='Run linters.',
         parser=setup_argument_parser)
     def lint(self, *runargs, **lintargs):
         """Run linters."""
         from mozlint import cli
         lintargs['exclude'] = ['obj*']
         cli.SEARCH_PATHS.append(here)
+        self._activate_virtualenv()
         return cli.run(*runargs, **lintargs)
 
     @Command('eslint', category='devenv',
              description='Run eslint or help configure eslint for optimal development.')
     @CommandArgument('paths', default=None, nargs='*',
                      help="Paths to file or directories to lint, like "
                           "'browser/components/loop' Defaults to the "
                           "current directory if not given.")