Bug 1494069 - [mozlint] Append global 'excludes' to each linter at parse time, r=rwood
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 16 Oct 2018 21:04:14 +0000
changeset 500145 53abbe1d64d4c976e07be000b2525a75b07f6c0a
parent 500144 9c587734513ad0366938cb19b378aaa15b28d209
child 500146 b9beb91e999214dfce51b7829b1f28de6b4e5581
push id1864
push userffxbld-merge
push dateMon, 03 Dec 2018 15:51:40 +0000
treeherdermozilla-release@f040763d99ad [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1494069 - [mozlint] Append global 'excludes' to each linter at parse time, r=rwood Right now there are excludes defined in the linter definition (via the .yml files), as well as excludes defined in lintargs (via the mach_commands.py). This is a minor simplification that extends each linter definition's local excludes with the global ones right off the bat. This just makes it a bit easier to keep track of. Depends on D5863 Differential Revision: https://phabricator.services.mozilla.com/D8844
--- a/python/mozlint/mozlint/roller.py
+++ b/python/mozlint/mozlint/roller.py
@@ -4,16 +4,17 @@
 from __future__ import absolute_import, print_function, unicode_literals
 import os
 import signal
 import sys
 import traceback
 from concurrent.futures import ProcessPoolExecutor
+from itertools import chain
 from math import ceil
 from multiprocessing import cpu_count
 from multiprocessing.queues import Queue
 from subprocess import CalledProcessError
 import mozpack.path as mozpath
 from mozversioncontrol import get_repository_object, MissingUpstreamRepo, InvalidRepoPath
@@ -88,42 +89,45 @@ class LintRoller(object):
     :param root: Path to which relative paths will be joined. If
                  unspecified, root will either be determined from
                  version control or cwd.
     :param lintargs: Arguments to pass to the underlying linter(s).
     MAX_PATHS_PER_JOB = 50  # set a max size to prevent command lines that are too long on Windows
-    def __init__(self, root, **lintargs):
+    def __init__(self, root, exclude=None, **lintargs):
         self.parse = Parser(root)
             self.vcs = get_repository_object(root)
         except InvalidRepoPath:
             self.vcs = None
         self.linters = []
         self.lintargs = lintargs
         self.lintargs['root'] = root
         # result state
         self.result = ResultSummary()
         self.root = root
+        self.exclude = exclude or []
     def read(self, paths):
         """Parse one or more linters and add them to the registry.
         :param paths: A path or iterable of paths to linter definitions.
         if isinstance(paths, basestring):
             paths = (paths,)
-        for path in paths:
-            self.linters.extend(self.parse(path))
+        for linter in chain(*[self.parse(p) for p in paths]):
+            # Add in our global excludes
+            linter.setdefault('exclude', []).extend(self.exclude)
+            self.linters.append(linter)
     def setup(self):
         """Run setup for applicable linters"""
         if not self.linters:
             raise LintersNotConfigured
         for linter in self.linters:
             if 'setup' not in linter:
--- a/python/mozlint/mozlint/types.py
+++ b/python/mozlint/mozlint/types.py
@@ -31,20 +31,20 @@ class BaseType(object):
                          the definition, but passed in by a consumer.
         :returns: A list of :class:`~result.Issue` objects.
         if lintargs.get('use_filters', True):
             paths, exclude = filterpaths(
-                config.get('exclude', []) + lintargs.get('exclude', []),
+                config.get('exclude', []),
                 config.get('extensions', []),
-            lintargs['exclude'] = exclude
+            config['exclude'] = exclude
         if not paths:
         if self.batch:
             return self._lint(paths, config, **lintargs)
         errors = []
@@ -75,17 +75,17 @@ class LineType(BaseType):
     def _lint_dir(self, path, config, **lintargs):
         if not config.get('extensions'):
             patterns = ['**']
             patterns = ['**/*.{}'.format(e) for e in config['extensions']]
-        exclude = [os.path.relpath(e, path) for e in lintargs.get('exclude', [])]
+        exclude = [os.path.relpath(e, path) for e in config.get('exclude', [])]
         finder = FileFinder(path, ignore=exclude)
         errors = []
         for pattern in patterns:
             for p, f in finder.find(pattern):
                 errors.extend(self._lint(os.path.join(path, p), config, **lintargs))
         return errors
--- a/python/mozlint/test/test_roller.py
+++ b/python/mozlint/test/test_roller.py
@@ -83,18 +83,17 @@ def test_roll_catch_exception(lint, lint
     lint.roll(files)  # assert not raises
     out, err = capfd.readouterr()
     assert 'LintException' in err
 def test_roll_with_global_excluded_path(lint, linters, files):
-    lint.lintargs.update({'exclude': ['**/foobar.js']})
+    lint.exclude = ['**/foobar.js']
     lint.read(linters('string', 'regex', 'external'))
     result = lint.roll(files)
     assert len(result.issues) == 0
     assert result.failed == set([])
 def test_roll_with_local_excluded_path(lint, linters, files):
--- a/tools/lint/python/compat.py
+++ b/tools/lint/python/compat.py
@@ -53,17 +53,17 @@ def run_linter(python, paths, config, **
         # If we're in automation, this is fatal. Otherwise, the warning in the
         # setup method was already printed.
         if 'MOZ_AUTOMATION' in os.environ:
             return 1
         return []
     root = lintargs['root']
     pattern = "**/*.py"
-    exclude = [mozpath.join(root, e) for e in lintargs.get('exclude', [])]
+    exclude = [mozpath.join(root, e) for e in config.get('exclude', [])]
     files = []
     for path in paths:
         path = mozpath.normsep(path)
         if os.path.isfile(path):
         ignore = [e[len(path):].lstrip('/') for e in exclude
--- a/tools/lint/python/flake8.py
+++ b/tools/lint/python/flake8.py
@@ -124,18 +124,18 @@ def lint(paths, config, **lintargs):
     fix_cmdargs = [
         os.path.join(bindir, 'autopep8'),
         '--global-config', os.path.join(lintargs['root'], '.flake8'),
         '--in-place', '--recursive',
-    if 'exclude' in lintargs:
-        fix_cmdargs.extend(['--exclude', ','.join(lintargs['exclude'])])
+    if config.get('exclude'):
+        fix_cmdargs.extend(['--exclude', ','.join(config['exclude'])])
     # Run any paths with a .flake8 file in the directory separately so
     # it gets picked up. This means only .flake8 files that live in
     # directories that are explicitly included will be considered.
     # See bug 1277851
     paths_by_config = defaultdict(list)
     for path in paths:
         configs = get_ancestors_by_name('.flake8', path, lintargs['root'])