Bug 1494069 - [mozlint] Append global 'excludes' to each linter at parse time, r=rwood
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 16 Oct 2018 21:04:14 +0000
changeset 497434 53abbe1d64d4c976e07be000b2525a75b07f6c0a
parent 497433 9c587734513ad0366938cb19b378aaa15b28d209
child 497435 b9beb91e999214dfce51b7829b1f28de6b4e5581
push id9996
push userarchaeopteryx@coole-files.de
push dateThu, 18 Oct 2018 18:37:15 +0000
treeherdermozilla-beta@8efe26839243 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrwood
bugs1494069
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1494069 - [mozlint] Append global 'excludes' to each linter at parse time, r=rwood Right now there are excludes defined in the linter definition (via the .yml files), as well as excludes defined in lintargs (via the mach_commands.py). This is a minor simplification that extends each linter definition's local excludes with the global ones right off the bat. This just makes it a bit easier to keep track of. Depends on D5863 Differential Revision: https://phabricator.services.mozilla.com/D8844
python/mozlint/mozlint/roller.py
python/mozlint/mozlint/types.py
python/mozlint/test/test_roller.py
tools/lint/python/compat.py
tools/lint/python/flake8.py
--- a/python/mozlint/mozlint/roller.py
+++ b/python/mozlint/mozlint/roller.py
@@ -4,16 +4,17 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os
 import signal
 import sys
 import traceback
 from concurrent.futures import ProcessPoolExecutor
+from itertools import chain
 from math import ceil
 from multiprocessing import cpu_count
 from multiprocessing.queues import Queue
 from subprocess import CalledProcessError
 
 import mozpack.path as mozpath
 from mozversioncontrol import get_repository_object, MissingUpstreamRepo, InvalidRepoPath
 
@@ -88,42 +89,45 @@ class LintRoller(object):
 
     :param root: Path to which relative paths will be joined. If
                  unspecified, root will either be determined from
                  version control or cwd.
     :param lintargs: Arguments to pass to the underlying linter(s).
     """
     MAX_PATHS_PER_JOB = 50  # set a max size to prevent command lines that are too long on Windows
 
-    def __init__(self, root, **lintargs):
+    def __init__(self, root, exclude=None, **lintargs):
         self.parse = Parser(root)
         try:
             self.vcs = get_repository_object(root)
         except InvalidRepoPath:
             self.vcs = None
 
         self.linters = []
         self.lintargs = lintargs
         self.lintargs['root'] = root
 
         # result state
         self.result = ResultSummary()
 
         self.root = root
+        self.exclude = exclude or []
 
     def read(self, paths):
         """Parse one or more linters and add them to the registry.
 
         :param paths: A path or iterable of paths to linter definitions.
         """
         if isinstance(paths, basestring):
             paths = (paths,)
 
-        for path in paths:
-            self.linters.extend(self.parse(path))
+        for linter in chain(*[self.parse(p) for p in paths]):
+            # Add in our global excludes
+            linter.setdefault('exclude', []).extend(self.exclude)
+            self.linters.append(linter)
 
     def setup(self):
         """Run setup for applicable linters"""
         if not self.linters:
             raise LintersNotConfigured
 
         for linter in self.linters:
             if 'setup' not in linter:
--- a/python/mozlint/mozlint/types.py
+++ b/python/mozlint/mozlint/types.py
@@ -31,20 +31,20 @@ class BaseType(object):
                          the definition, but passed in by a consumer.
         :returns: A list of :class:`~result.Issue` objects.
         """
         if lintargs.get('use_filters', True):
             paths, exclude = filterpaths(
                 lintargs['root'],
                 paths,
                 config['include'],
-                config.get('exclude', []) + lintargs.get('exclude', []),
+                config.get('exclude', []),
                 config.get('extensions', []),
             )
-            lintargs['exclude'] = exclude
+            config['exclude'] = exclude
 
         if not paths:
             return
 
         if self.batch:
             return self._lint(paths, config, **lintargs)
 
         errors = []
@@ -75,17 +75,17 @@ class LineType(BaseType):
         pass
 
     def _lint_dir(self, path, config, **lintargs):
         if not config.get('extensions'):
             patterns = ['**']
         else:
             patterns = ['**/*.{}'.format(e) for e in config['extensions']]
 
-        exclude = [os.path.relpath(e, path) for e in lintargs.get('exclude', [])]
+        exclude = [os.path.relpath(e, path) for e in config.get('exclude', [])]
         finder = FileFinder(path, ignore=exclude)
 
         errors = []
         for pattern in patterns:
             for p, f in finder.find(pattern):
                 errors.extend(self._lint(os.path.join(path, p), config, **lintargs))
         return errors
 
--- a/python/mozlint/test/test_roller.py
+++ b/python/mozlint/test/test_roller.py
@@ -83,18 +83,17 @@ def test_roll_catch_exception(lint, lint
     lint.read(linters('raises'))
 
     lint.roll(files)  # assert not raises
     out, err = capfd.readouterr()
     assert 'LintException' in err
 
 
 def test_roll_with_global_excluded_path(lint, linters, files):
-    lint.lintargs.update({'exclude': ['**/foobar.js']})
-
+    lint.exclude = ['**/foobar.js']
     lint.read(linters('string', 'regex', 'external'))
     result = lint.roll(files)
 
     assert len(result.issues) == 0
     assert result.failed == set([])
 
 
 def test_roll_with_local_excluded_path(lint, linters, files):
--- a/tools/lint/python/compat.py
+++ b/tools/lint/python/compat.py
@@ -53,17 +53,17 @@ def run_linter(python, paths, config, **
         # If we're in automation, this is fatal. Otherwise, the warning in the
         # setup method was already printed.
         if 'MOZ_AUTOMATION' in os.environ:
             return 1
         return []
 
     root = lintargs['root']
     pattern = "**/*.py"
-    exclude = [mozpath.join(root, e) for e in lintargs.get('exclude', [])]
+    exclude = [mozpath.join(root, e) for e in config.get('exclude', [])]
     files = []
     for path in paths:
         path = mozpath.normsep(path)
         if os.path.isfile(path):
             files.append(path)
             continue
 
         ignore = [e[len(path):].lstrip('/') for e in exclude
--- a/tools/lint/python/flake8.py
+++ b/tools/lint/python/flake8.py
@@ -124,18 +124,18 @@ def lint(paths, config, **lintargs):
     ]
 
     fix_cmdargs = [
         os.path.join(bindir, 'autopep8'),
         '--global-config', os.path.join(lintargs['root'], '.flake8'),
         '--in-place', '--recursive',
     ]
 
-    if 'exclude' in lintargs:
-        fix_cmdargs.extend(['--exclude', ','.join(lintargs['exclude'])])
+    if config.get('exclude'):
+        fix_cmdargs.extend(['--exclude', ','.join(config['exclude'])])
 
     # Run any paths with a .flake8 file in the directory separately so
     # it gets picked up. This means only .flake8 files that live in
     # directories that are explicitly included will be considered.
     # See bug 1277851
     paths_by_config = defaultdict(list)
     for path in paths:
         configs = get_ancestors_by_name('.flake8', path, lintargs['root'])