Bug 1515746 - [flake8] Unsupport subdir .flake8 files and use new 'per-file-ignores' config instead, r=egao
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 07 Feb 2019 20:17:02 +0000
changeset 458190 f96c1460ffc09ef61bf3e2c93bfda21cdb86fb2f
parent 458189 3c7c50fba28399ae0d1a3812e5088cde55aacd8e
child 458191 933722dee47bb2a80d4b4fd32966f97448c6c79c
push id111775
push useropoprus@mozilla.com
push dateFri, 08 Feb 2019 10:16:11 +0000
treeherdermozilla-inbound@fe34a6921349 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersegao
bugs1515746, 1367092
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1515746 - [flake8] Unsupport subdir .flake8 files and use new 'per-file-ignores' config instead, r=egao This removes all .flake8 files except for the one at the root of the repo. Instead we use the new 'per-file-ignores' config introduced in 3.7. To ignore specific errors in a subdirectory, add a line like this to the root .flake8: [per-file-ignores] path/to/subdir/*: E100, F200, ... The reasons for this change are: 1. Unblock flake8 blacklist (bug 1367092). 2. Simplify configuration and code. 3. Encourage more consistent styling. 4. Improve performance. 5. Greater editor consistency. Differential Revision: https://phabricator.services.mozilla.com/D18354
.flake8
build/moz.configure/.flake8
ipc/ipdl/.flake8
security/manager/.flake8
testing/firefox-ui/.flake8
testing/marionette/client/.flake8
testing/marionette/harness/.flake8
testing/marionette/puppeteer/.flake8
testing/mozharness/configs/.flake8
tools/lint/python/flake8.py
tools/lint/test/test_flake8.py
--- a/.flake8
+++ b/.flake8
@@ -13,23 +13,32 @@ exclude =
     js/*.configure,
     gfx/angle/,
     gfx/harfbuzz,
     glx/skia/,
     memory/moz.configure,
     mobile/android/*.configure,
     node_modules,
     security/nss/,
+    testing/firefox-ui/**/__init__.py,
+    testing/marionette/**/__init__.py,
+    testing/marionette/harness/marionette_harness/runner/mixins,
+    testing/marionette/harness/marionette_harness/tests,
     testing/mochitest/pywebsocket,
+    testing/mozharness/configs/test/test_malformed.py,
     tools/lint/test/files,
     tools/infer/test/*.configure,
     tools/crashreporter/*.configure,
 
 # See:
 #   - http://flake8.pycqa.org/en/latest/user/error-codes.html
 #   - http://pep8.readthedocs.io/en/latest/intro.html#configuration
 ignore =
     # These should be triaged and either fixed or moved to the list below.
     F632, F633, F811, E117, W504, W605, W606,
     # These are intentionally disabled (not necessarily for good reason).
     #   F723: syntax error in type comment
     #       text contains quotes which breaks our custom JSON formatter
     F723, E121, E123, E126, E129, E133, E226, E241, E242, E402, E704, E741, W503,
+
+per-file-ignores =
+    ipc/ipdl/*: F403, F405
+    testing/mozharness/configs/*: E124, E127, E128, E131, E231, E261, E265, E266, E501, W391
deleted file mode 100644
--- a/build/moz.configure/.flake8
+++ /dev/null
@@ -1,5 +0,0 @@
-[flake8]
-# F821: undefined name
-#       This is the moz.configure style
-ignore = F821
-
deleted file mode 100644
--- a/ipc/ipdl/.flake8
+++ /dev/null
@@ -1,5 +0,0 @@
-[flake8]
-# See http://pep8.readthedocs.io/en/latest/intro.html#configuration
-ignore = E121, E123, E126, E129, E133, E226, E241, E242, E704, W503, E402, E741, F405, F403, W504
-max-line-length = 99
-
deleted file mode 100644
--- a/security/manager/.flake8
+++ /dev/null
@@ -1,14 +0,0 @@
-[flake8]
-# E128: continuation line under-indented for visual indent.
-#       Best visual indent typically determined based on context.
-# E261: at least two spaces before inline comment.
-#       Not PSM style.
-# E302: expected 2 blank lines, found 1.
-#       Not really PSM style.
-ignore = E128,E261,E302,W605,W504
-
-# PSM style is to stick close to 80 chars, but 4 space indentation means we
-# sometimes need more space.
-max-line-length = 100
-
-filename = *.py
deleted file mode 100644
--- a/testing/firefox-ui/.flake8
+++ /dev/null
@@ -1,3 +0,0 @@
-[flake8]
-exclude =
-    __init__.py,
deleted file mode 100644
--- a/testing/marionette/client/.flake8
+++ /dev/null
@@ -1,5 +0,0 @@
-[flake8]
-exclude =
-    __init__.py,
-    disti/*,
-    build/*,
deleted file mode 100644
--- a/testing/marionette/harness/.flake8
+++ /dev/null
@@ -1,7 +0,0 @@
-[flake8]
-exclude =
-    __init__.py,
-    disti/*,
-    build/*,
-    marionette_harness/runner/mixins/*,
-    marionette_harness/tests/*,
deleted file mode 100644
--- a/testing/marionette/puppeteer/.flake8
+++ /dev/null
@@ -1,3 +0,0 @@
-[flake8]
-exclude =
-    __init__.py,
deleted file mode 100644
--- a/testing/mozharness/configs/.flake8
+++ /dev/null
@@ -1,33 +0,0 @@
-[flake8]
-ignore =
-    # From root
-    E121, E123, E126, E129, E133, E226, E241, E242, E704, W503, E402, W605, W504,
-    # The following errors should be fixed eventually
-    # line too long
-    E501,
-    # at least two spaces before inline comment
-    E261,
-    # continuation line under-indented for visual indent
-    E128,
-    # whitespace before ':'
-    E203,
-    # blank line at end of file
-    W391,
-    # multiple statements on one line
-    E702,
-    # closing bracket does not match visual indentation
-    E124,
-    # missing whitespace after ':'
-    E231,
-    # continuation line over-indented for visual indent
-    E127,
-    # too many leading '#' for block comment
-    E266,
-    # comment should start with '# '
-    E262, E265,
-    # continuation line unaligned for hanging indent
-    E131,
-    # continuation line missing indentation or outdented
-    E122,
-exclude =
-    test/test_malformed.py
--- a/tools/lint/python/flake8.py
+++ b/tools/lint/python/flake8.py
@@ -3,23 +3,21 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import json
 import os
 import platform
 import signal
 import subprocess
 import sys
-from collections import defaultdict
 
 from mozprocess import ProcessHandlerMixin
 
 from mozlint import result
 from mozlint.util import pip
-from mozlint.pathutils import get_ancestors_by_name
 
 
 here = os.path.abspath(os.path.dirname(__file__))
 FLAKE8_REQUIREMENTS_PATH = os.path.join(here, 'flake8_requirements.txt')
 
 FLAKE8_NOT_FOUND = """
 Could not find flake8! Install flake8 and try again.
 
@@ -90,68 +88,49 @@ class Flake8Process(ProcessHandlerMixin)
     def run(self, *args, **kwargs):
         # flake8 seems to handle SIGINT poorly. Handle it here instead
         # so we can kill the process without a cryptic traceback.
         orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
         ProcessHandlerMixin.run(self, *args, **kwargs)
         signal.signal(signal.SIGINT, orig)
 
 
-def run_process(config, cmd):
-    proc = Flake8Process(config, cmd)
-    proc.run()
-    try:
-        proc.wait()
-    except KeyboardInterrupt:
-        proc.kill()
-        return 1
-
-
 def setup(root):
     if not pip.reinstall_program(FLAKE8_REQUIREMENTS_PATH):
         print(FLAKE8_INSTALL_ERROR)
         return 1
 
 
 def lint(paths, config, **lintargs):
     # TODO don't store results in a global
     global results
     results = []
 
+    config_path = os.path.join(lintargs['root'], '.flake8')
     cmdargs = [
         os.path.join(bindir, 'flake8'),
+        '--config', config_path,
         '--format', '{"path":"%(path)s","lineno":%(row)s,'
                     '"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}',
         '--filename', ','.join(['*.{}'.format(e) for e in config['extensions']]),
     ]
 
-    fix_cmdargs = [
-        os.path.join(bindir, 'autopep8'),
-        '--global-config', os.path.join(lintargs['root'], '.flake8'),
-        '--in-place', '--recursive',
-    ]
+    if lintargs.get('fix'):
+        fix_cmdargs = [
+            os.path.join(bindir, 'autopep8'),
+            '--global-config', config_path,
+            '--in-place', '--recursive',
+        ]
 
-    if config.get('exclude'):
-        fix_cmdargs.extend(['--exclude', ','.join(config['exclude'])])
+        if config.get('exclude'):
+            fix_cmdargs.extend(['--exclude', ','.join(config['exclude'])])
 
-    # Run any paths with a .flake8 file in the directory separately so
-    # it gets picked up. This means only .flake8 files that live in
-    # directories that are explicitly included will be considered.
-    # See bug 1277851
-    paths_by_config = defaultdict(list)
-    for path in paths:
-        configs = get_ancestors_by_name('.flake8', path, lintargs['root'])
-        paths_by_config[os.pathsep.join(configs) if configs else 'default'].append(path)
+        subprocess.call(fix_cmdargs + paths)
 
-    for configs, paths in paths_by_config.items():
-        if lintargs.get('fix'):
-            subprocess.call(fix_cmdargs + paths)
-
-        cmd = cmdargs[:]
-        if configs != 'default':
-            configs = reversed(configs.split(os.pathsep))
-            cmd.extend(['--append-config={}'.format(c) for c in configs])
-
-        cmd.extend(paths)
-        if run_process(config, cmd):
-            break
+    proc = Flake8Process(config, cmdargs + paths)
+    proc.run()
+    try:
+        proc.wait()
+    except KeyboardInterrupt:
+        proc.kill()
+        return 1
 
     return results
--- a/tools/lint/test/test_flake8.py
+++ b/tools/lint/test/test_flake8.py
@@ -13,24 +13,21 @@ def test_lint_single_file(lint, paths):
     assert results[1].rule == 'E501'
     assert results[1].lineno == 5
 
     # run lint again to make sure the previous results aren't counted twice
     results = lint(paths('bad.py'))
     assert len(results) == 2
 
 
-def test_lint_custom_config(lint, paths):
+def test_lint_custom_config_ignored(lint, paths):
     results = lint(paths('custom'))
-    assert len(results) == 0
+    assert len(results) == 2
 
     results = lint(paths('custom/good.py'))
-    assert len(results) == 0
-
-    results = lint(paths('custom', 'bad.py'))
     assert len(results) == 2
 
 
 def test_lint_fix(lint, create_temp_file):
     contents = """
 import distutils
 
 def foobar():