Bug 1566097 - Setup rstcheck linter instead of restructuredtext-lint. r=ahal
authorchampionshuttler <shivams2799@gmail.com>
Tue, 23 Jul 2019 21:30:10 +0000
changeset 483912 35c99c4a67383e4b78a84c8d744ee27a3bb0af25
parent 483911 4d0c22c47c7fa0a54c96b58008b3bd516b52b6d4
child 483913 c2a690d8461ca420f0ac9e864c07157bd89ea52a
push id36336
push usermalexandru@mozilla.com
push dateWed, 24 Jul 2019 09:54:28 +0000
treeherdermozilla-central@5585edba8fdb [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1566097
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1566097 - Setup rstcheck linter instead of restructuredtext-lint. r=ahal Differential Revision: https://phabricator.services.mozilla.com/D38339
tools/lint/docs/linters/rstlinter.rst
tools/lint/rst.yml
tools/lint/rst/__init__.py
tools/lint/rst/requirements.txt
--- a/tools/lint/docs/linters/rstlinter.rst
+++ b/tools/lint/docs/linters/rstlinter.rst
@@ -1,12 +1,12 @@
 RST Linter
 ==========
 
-`Restructuredtext lint`_ is a popular linter for restructuredtext.
+`rstcheck`_ is a popular linter for restructuredtext.
 
 
 Run Locally
 -----------
 
 The mozlint integration of rst linter can be run using mach:
 
 .. parsed-literal::
@@ -17,9 +17,9 @@ The mozlint integration of rst linter ca
 Configuration
 -------------
 
 All directories will have rst linter run against them.
 If you wish to exclude a subdirectory of an included one, you can add it to the ``exclude``
 directive.
 
 
-.. _Restructuredtext lint: https://github.com/twolfson/restructuredtext-lint
+.. _rstcheck: https://github.com/myint/rstcheck
\ No newline at end of file
--- a/tools/lint/rst.yml
+++ b/tools/lint/rst.yml
@@ -1,12 +1,17 @@
 ---
 rst:
     description: RST linter
-    include:
-        - tools/lint/docs/usage.rst
-    exclude: []
+    include: [.]
+    exclude:
+        - toolkit/components/telemetry/docs/
+        - toolkit/components/extensions/
+        - taskcluster/docs/
+        - tools/tryselect/docs/
+        - browser/docs/
     extensions:
         - rst
     support-files:
         - 'tools/lint/rst/**'
     type: external
     payload: rst:lint
+    setup: rst:setup
--- a/tools/lint/rst/__init__.py
+++ b/tools/lint/rst/__init__.py
@@ -1,92 +1,102 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function
 
 import os
 import subprocess
-import json
 
 from mozlint import result
+from mozlint.pathutils import expand_exclusions
+from mozlint.util import pip
+from mozfile import which
 
 # Error Levels
 # (0, 'debug')
 # (1, 'info')
 # (2, 'warning')
 # (3, 'error')
 # (4, 'severe')
 
 abspath = os.path.abspath(os.path.dirname(__file__))
-rstlint_requirements_file = os.path.join(abspath, 'requirements.txt')
+rstcheck_requirements_file = os.path.join(abspath, 'requirements.txt')
+
+results = []
 
-RSTLINT_INSTALL_ERROR = """
-Unable to install required version of restructuredtext_lint and docutils
+RSTCHECK_NOT_FOUND = """
+Could not find rstcheck! Install rstcheck and try again.
+
+    $ pip install -U --require-hashes -r {}
+""".strip().format(rstcheck_requirements_file)
+
+RSTCHECK_INSTALL_ERROR = """
+Unable to install required version of rstcheck
 Try to install it manually with:
     $ pip install -U --require-hashes -r {}
-""".strip().format(rstlint_requirements_file)
+""".strip().format(rstcheck_requirements_file)
+
+
+def setup(root, **lintargs):
+    if not pip.reinstall_program(rstcheck_requirements_file):
+        print(RSTCHECK_INSTALL_ERROR)
+        return 1
 
 
-def pip_installer(*args):
+def get_rstcheck_binary():
     """
-    Helper function that runs pip with subprocess
+    Returns the path of the first rstcheck binary available
+    if not found returns None
     """
-    try:
-        subprocess.check_output(['pip'] + list(args),
-                                stderr=subprocess.STDOUT)
-        return True
-    except subprocess.CalledProcessError as e:
-        print(e.output)
-        return False
+    binary = os.environ.get('RSTCHECK')
+    if binary:
+        return binary
+
+    return which('rstcheck')
 
 
-def install_rstlint():
-    """
-    Try to install rstlint and docutils at the target version, returns True on success
-    otherwise prints the otuput of the pip command and returns False
-    """
-    if pip_installer('install', '-U',
-                     '--require-hashes', '-r',
-                     rstlint_requirements_file):
-        return True
+def parse_with_split(errors):
 
-    return False
+    filtered_output = errors.split(":")
+    filename = filtered_output[0]
+    lineno = filtered_output[1]
+    idx = filtered_output[2].index(") ")
+    level = filtered_output[2][0:idx].split("/")[1]
+    message = filtered_output[2][idx+2:].split("\n")[0]
+
+    return filename, lineno, level, message
 
 
 def lint(files, config, **lintargs):
 
-    if not install_rstlint():
-        print(RSTLINT_INSTALL_ERROR)
-        return 1
-
-    config = config.copy()
     config['root'] = lintargs['root']
-
-    cmdargs = [
-        'rst-lint',
-        '--format=json',
-    ] + files
-
-    proc = subprocess.Popen(cmdargs, stdout=subprocess.PIPE, env=os.environ)
-    output = proc.communicate()[0]
+    paths = expand_exclusions(files, config, config['root'])
+    paths = list(paths)
+    chunk_size = 50
+    binary = get_rstcheck_binary()
 
-    # all passed
-    if not output:
-        return []
-
-    results = []
-
-    for i in (json.loads(output)):
-
-        # create a dictionary to append with mozlint.results
-        res = {
-            'path': i['source'],
-            'message': i['message'],
-            'lineno': i['line'],
-            'level': i['type'].lower(),
-            'rule': i['level'],
-        }
-
-        results.append(result.from_config(config, **res))
+    while paths:
+        cmdargs = [
+            binary,
+        ] + paths[:chunk_size]
+        proc = subprocess.Popen(
+            cmdargs, stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=os.environ
+            )
+        all_errors = proc.communicate()[1]
+        for errors in all_errors.split("\n"):
+            if len(errors) > 1:
+                filename, lineno, level, message = parse_with_split(errors)
+                if int(level) < 3:
+                    continue
+                res = {
+                    'path': filename,
+                    'message': message,
+                    'lineno': lineno,
+                    'level': "error" if int(level) >= 3 else "warning",
+                }
+                results.append(result.from_config(config, **res))
+        paths = paths[chunk_size:]
 
     return results
--- a/tools/lint/rst/requirements.txt
+++ b/tools/lint/rst/requirements.txt
@@ -1,6 +1,6 @@
-restructuredtext-lint==1.3.0 \
-    --hash=sha256:97b3da356d5b3a8514d8f1f9098febd8b41463bed6a1d9f126cf0a048b6fd908
 docutils==0.14 \
     --hash=sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6 \
     --hash=sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274 \
     --hash=sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6
+rstcheck==3.3.1 \
+    --hash=sha256:92c4f79256a54270e0402ba16a2f92d0b3c15c8f4410cb9c57127067c215741f