author Andrew Halberstadt <>
Wed, 23 Aug 2017 11:28:28 -0400
changeset 376964 f384a3acd01687f4f8cc55333298aebb02c33532
parent 373743 cc4c01794114b8f756f916d00109bf56ce5d2402
child 377819 0145883aea3a74740298a2dc3072e4a4e80c8570
permissions -rw-r--r--
Bug 1392795 - [yamllint] Group paths to lint by their closest config and run each config group separately, r=dustin This makes configuration files for yamllint work a bit better. It's still not perfect, but it's an improvement on the current situation. MozReview-Commit-ID: IKxgQm1a7bP

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at

import re
import os
import signal
import subprocess
from collections import defaultdict

import which
from mozprocess import ProcessHandlerMixin

from mozlint import result

here = os.path.abspath(os.path.dirname(__file__))
YAMLLINT_REQUIREMENTS_PATH = os.path.join(here, 'yamllint_requirements.txt')

Unable to install correct version of yamllint
Try to install it manually with:
    $ pip install -U --require-hashes -r {}

YAMLLINT_FORMAT_REGEX = re.compile(r'(.*):(.*):(.*): \[(error|warning)\] (.*) \((.*)\)$')

results = []

class YAMLLintProcess(ProcessHandlerMixin):
    def __init__(self, config, *args, **kwargs):
        self.config = config
        kwargs['processOutputLine'] = [self.process_line]
        ProcessHandlerMixin.__init__(self, *args, **kwargs)

    def process_line(self, line):
            match = YAMLLINT_FORMAT_REGEX.match(line)
            abspath, line, col, level, message, code = match.groups()
        except AttributeError:
            print('Unable to match yaml regex against output: {}'.format(line))

        res = {'path': os.path.relpath(abspath, self.config['root']),
               'message': message,
               'level': level,
               'lineno': line,
               'column': col,
               'rule': code,

        results.append(result.from_config(self.config, **res))

    def run(self, *args, **kwargs):
        # protect against poor SIGINT handling. Handle it here instead
        # so we can kill the process without a cryptic traceback.
        orig = signal.signal(signal.SIGINT, signal.SIG_IGN), *args, **kwargs)
        signal.signal(signal.SIGINT, orig)

def get_yamllint_binary():
    Returns the path of the first yamllint binary available
    if not found returns None
    binary = os.environ.get('YAMLLINT')
    if binary:
        return binary

        return which.which('yamllint')
    except which.WhichError:
        return None

def _run_pip(*args):
    Helper function that runs pip with subprocess
        subprocess.check_output(['pip'] + list(args),
        return True
    except subprocess.CalledProcessError as e:
        return False

def reinstall_yamllint():
    Try to install yamllint at the target version, returns True on success
    otherwise prints the otuput of the pip command and returns False
    if _run_pip('install', '-U',
                '--require-hashes', '-r',
        return True

    return False

def run_process(config, cmd):
    proc = YAMLLintProcess(config, cmd)
    except KeyboardInterrupt:

def gen_yamllint_args(cmdargs, paths=None, conf_file=None):
    args = cmdargs[:]
    if isinstance(paths, basestring):
        paths = [paths]
    if conf_file and conf_file != 'default':
        return args + ['-c', conf_file] + paths
    return args + paths

def ancestors(path):
    while path:
        yield path
        (path, child) = os.path.split(path)
        if child == "":

def get_relevant_configs(name, path, root):
    """Returns a list of configuration files that exist in `path`'s ancestors,
    sorted from closest->furthest.
    configs = []
    for path in ancestors(path):
        if path == root:

        config = os.path.join(path, name)
        if os.path.isfile(config):
    return configs

def lint(files, config, **lintargs):
    if not reinstall_yamllint():
        return 1

    binary = get_yamllint_binary()

    cmdargs = [
        '-f', 'parsable'

    config = config.copy()
    config['root'] = lintargs['root']

    # Run any paths with a .yamllint file in the directory separately so
    # it gets picked up. This means only .yamllint files that live in
    # directories that are explicitly included will be considered.
    paths_by_config = defaultdict(list)
    for f in files:
        conf_files = get_relevant_configs('.yamllint', f, config['root'])
        paths_by_config[conf_files[0] if conf_files else 'default'].append(f)

    for conf_file, paths in paths_by_config.items():
        run_process(config, gen_yamllint_args(cmdargs, conf_file=conf_file, paths=paths))

    return results