Bug 1623024 - mozlint: Add pylint as new linter r=linter-reviewers,ahal
authorSylvestre Ledru <sledru@mozilla.com>
Thu, 18 Jun 2020 20:04:50 +0000
changeset 536379 ba8087e38e6c4c4dabe2db96aded343fb1a86fcf
parent 536378 512cb9dac1bb5888ac6d2caf422d172ed9e9f9e0
child 536380 2ba0929ca2b552e0450a80bd3aaa508058e0312d
push id37520
push userdluca@mozilla.com
push dateFri, 19 Jun 2020 04:04:08 +0000
treeherdermozilla-central@d1a4f9157858 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerslinter-reviewers, ahal
bugs1623024
milestone79.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1623024 - mozlint: Add pylint as new linter r=linter-reviewers,ahal Differential Revision: https://phabricator.services.mozilla.com/D79076
docs/code-quality/index.rst
docs/code-quality/lint/linters/pylint.rst
tools/lint/pylint.yml
tools/lint/python/pylint.py
tools/lint/python/pylint_requirements.txt
tools/lint/test/files/pylint/bad.py
tools/lint/test/files/pylint/good.py
tools/lint/test/python.ini
tools/lint/test/test_pylint.py
--- a/docs/code-quality/index.rst
+++ b/docs/code-quality/index.rst
@@ -95,17 +95,22 @@ In this document, we try to list these a
      - Meta bug
      - More info
      - Upstream
    * - Flake8
      - Yes (with `autopep8 <https://github.com/hhatto/autopep8>`_)
      - `bug 1155970 <https://bugzilla.mozilla.org/show_bug.cgi?id=1155970>`_
      - :ref:`Flake8`
      - http://flake8.pycqa.org/
-   * - Python 2/3 compatibility check
+   * - pylint
+     -
+     - `bug 1623024 <https://bugzilla.mozilla.org/show_bug.cgi?id=1623024>`_
+     - :ref:`pylint`
+     - https://www.pylint.org/
+     * - Python 2/3 compatibility check
      -
      - `bug 1496527 <https://bugzilla.mozilla.org/show_bug.cgi?id=1496527>`_
      - :ref:`Python 2/3 compatibility check`
      -
 
 
 .. list-table:: Rust
    :widths: 20 20 20 20 20
new file mode 100644
--- /dev/null
+++ b/docs/code-quality/lint/linters/pylint.rst
@@ -0,0 +1,31 @@
+pylint
+======
+
+`pylint <https://www.pylint.org/>`__ is a popular linter for python developed by Logilab. It is now the default python
+linter in VS Code.
+
+Please note that we also have :ref:`Flake8` available as a linter.
+
+Run Locally
+-----------
+
+The mozlint integration of pylint can be run using mach:
+
+.. parsed-literal::
+
+    $ mach lint --linter pylint <file paths>
+
+
+
+Configuration
+-------------
+
+To enable pylint on new directory, add the path to the include
+section in the `pylint.yml <https://searchfox.org/mozilla-central/source/tools/lint/pylint.yml>`_ file.
+
+
+Sources
+-------
+
+* `Configuration (YAML) <https://searchfox.org/mozilla-central/source/tools/lint/pylint.yml>`_
+* `Source <https://searchfox.org/mozilla-central/source/tools/lint/python/pylint.py>`_
new file mode 100644
--- /dev/null
+++ b/tools/lint/pylint.yml
@@ -0,0 +1,24 @@
+---
+pylint:
+    description: A second Python linter
+    include:
+        - configure.py
+        - client.py
+        - security/
+        - accessible/
+        - docs/
+        - dom/base/
+        - mozglue/
+    exclude:
+        - dom/bindings/Codegen.py
+        - dom/bindings/Configuration.py
+        - security/manager/tools/crtshToIdentifyingStruct/crtshToIdentifyingStruct.py
+        - security/manager/ssl/tests/unit/test_content_signing/pysign.py
+        - security/ct/tests/gtest/createSTHTestData.py
+    extensions: ['py']
+    support-files:
+        - '**/.pylint'
+        - 'tools/lint/python/pylint*'
+    type: external
+    payload: python.pylint:lint
+    setup: python.pylint:setup
new file mode 100644
--- /dev/null
+++ b/tools/lint/python/pylint.py
@@ -0,0 +1,117 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+
+import signal
+import re
+
+from mozprocess import ProcessHandler
+
+from mozlint import result
+from mozlint.pathutils import expand_exclusions
+from mozlint.util import pip
+
+here = os.path.abspath(os.path.dirname(__file__))
+PYLINT_REQUIREMENTS_PATH = os.path.join(here, 'pylint_requirements.txt')
+
+PYLINT_NOT_FOUND = """
+Could not find pylint! Install pylint and try again.
+
+    $ pip install -U --require-hashes -r {}
+""".strip().format(PYLINT_REQUIREMENTS_PATH)
+
+
+PYLINT_INSTALL_ERROR = """
+Unable to install correct version of pylint
+Try to install it manually with:
+    $ pip install -U --require-hashes -r {}
+""".strip().format(PYLINT_REQUIREMENTS_PATH)
+
+
+class PylintProcess(ProcessHandler):
+    def __init__(self, config, *args, **kwargs):
+        self.config = config
+        kwargs["stream"] = False
+        kwargs["universal_newlines"] = True
+        ProcessHandler.__init__(self, *args, **kwargs)
+
+    def run(self, *args, **kwargs):
+        orig = signal.signal(signal.SIGINT, signal.SIG_IGN)
+        ProcessHandler.run(self, *args, **kwargs)
+        signal.signal(signal.SIGINT, orig)
+
+
+def setup(root, **lintargs):
+    if not pip.reinstall_program(PYLINT_REQUIREMENTS_PATH):
+        print(PYLINT_INSTALL_ERROR)
+        return 1
+
+
+def get_pylint_binary():
+    return "pylint"
+
+
+def run_process(config, cmd):
+    proc = PylintProcess(config, cmd)
+    proc.run()
+    try:
+        proc.wait()
+    except KeyboardInterrupt:
+        proc.kill()
+
+    return proc.output
+
+
+PYLINT_FORMAT_REGEX = re.compile(r'(.*):(.*): [(.*)] (.*)$')
+
+
+def parse_issues(log, config, issues_json, path):
+    results = []
+
+    try:
+        issues = json.loads(issues_json)
+    except json.decoder.JSONDecodeError:
+        log.debug("Could not parse the output:")
+        log.debug("pylint output: {}".format(issues_json))
+        return []
+
+    for issue in issues:
+        res = {
+            "path": issue["path"],
+            "level": issue["type"],
+            "lineno": issue["line"],
+            "column": issue["column"],
+            "message": issue["message"],
+            "rule": issue["message-id"],
+        }
+        results.append(result.from_config(config, **res))
+    return results
+
+
+def lint(paths, config, **lintargs):
+    log = lintargs['log']
+
+    binary = get_pylint_binary()
+
+    log = lintargs['log']
+    paths = list(expand_exclusions(paths, config, lintargs['root']))
+
+    cmd_args = [binary]
+    results = []
+
+    # list from https://code.visualstudio.com/docs/python/linting#_pylint
+    # And ignore a bit more elements
+    cmd_args += ["-fjson",
+                 "--disable=all",
+                 "--enable=F,E,unreachable,duplicate-key,unnecessary-semicolon,global-variable-not-assigned,unused-variable,binary-op-exception,bad-format-string,anomalous-backslash-in-string,bad-open-mode",  # NOQA: E501
+                 "--disable=import-error,no-member"]
+
+    base_command = cmd_args + paths
+    log.debug("Command: {}".format(' '.join(cmd_args)))
+    output = " ".join(run_process(config, base_command))
+    results = parse_issues(log, config, str(output), [])
+
+    return results
new file mode 100644
--- /dev/null
+++ b/tools/lint/python/pylint_requirements.txt
@@ -0,0 +1,61 @@
+pylint==2.5.3 \
+    --hash=sha256:7dd78437f2d8d019717dbf287772d0b2dbdfd13fc016aa7faa08d67bccc46adc \
+    --hash=sha256:d0ece7d223fe422088b0e8f13fa0a1e8eb745ebffcb8ed53d3e95394b6101a1c
+toml==0.10.1 \
+    --hash=sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f \
+    --hash=sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88
+mccabe==0.6.1 \
+    --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
+    --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f
+six==1.15.0 \
+    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
+    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
+wrapt==1.12.1 \
+    --hash=sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7
+lazy-object-proxy==1.5.0 \
+    --hash=sha256:0aef3fa29f7d1194d6f8a99382b1b844e5a14d3bc1ef82c3b1c4fb7e7e2019bc \
+    --hash=sha256:159ae2bbb4dc3ba506aeba868d14e56a754c0be402d1f0d7fdb264e0bdf2b095 \
+    --hash=sha256:161a68a427022bf13e249458be2cb8da56b055988c584d372a917c665825ae9a \
+    --hash=sha256:2d58f0e6395bf41087a383a48b06b42165f3b699f1aa41ba201db84ab77be63d \
+    --hash=sha256:311c9d1840042fc8e2dd80fc80272a7ea73e7646745556153c9cda85a4628b18 \
+    --hash=sha256:35c3ad7b7f7d5d4a54a80f0ff5a41ab186237d6486843f8dde00c42cfab33905 \
+    --hash=sha256:459ef557e669d0046fe2b92eb4822c097c00b5ef9d11df0f9bd7d4267acdfc52 \
+    --hash=sha256:4a50513b6be001b9b7be2c435478fe9669249c77c241813907a44cda1fcd03f4 \
+    --hash=sha256:51035b175740c44707694c521560b55b66da9d5a7c545cf22582bc02deb61664 \
+    --hash=sha256:96f2cdb35bdfda10e075f12892a42cff5179bbda698992b845f36c5e92755d33 \
+    --hash=sha256:a0aed261060cd0372abf08d16399b1224dbb5b400312e6b00f2b23eabe1d4e96 \
+    --hash=sha256:a6052c4c7d95de2345d9c58fc0fe34fff6c27a8ed8550dafeb18ada84406cc99 \
+    --hash=sha256:cbf1354292a4f7abb6a0188f74f5e902e4510ebad105be1dbc4809d1ed92f77e \
+    --hash=sha256:da82b2372f5ded8806eaac95b19af89a7174efdb418d4e7beb0c6ab09cee7d95 \
+    --hash=sha256:dd89f466c930d7cfe84c94b5cbe862867c88b269f23e5aa61d40945e0d746f54 \
+    --hash=sha256:e3183fbeb452ec11670c2d9bfd08a57bc87e46856b24d1c335f995239bedd0e1 \
+    --hash=sha256:e9a571e7168076a0d5ecaabd91e9032e86d815cca3a4bf0dafead539ef071aa5 \
+    --hash=sha256:ec6aba217d0c4f71cbe48aea962a382dedcd111f47b55e8b58d4aaca519bd360
+astroid==2.4.2 \
+    --hash=sha256:2f4078c2a41bf377eea06d71c9d2ba4eb8f6b1af2135bec27bbbb7d8f12bb703 \
+    --hash=sha256:bc58d83eb610252fd8de6363e39d4f1d0619c894b0ed24603b881c02e64c7386
+isort==4.3.21 \
+    --hash=sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1 \
+    --hash=sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd
+typed-ast==1.4.1 \
+    --hash=sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355 \
+    --hash=sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919 \
+    --hash=sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa \
+    --hash=sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652 \
+    --hash=sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75 \
+    --hash=sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01 \
+    --hash=sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d \
+    --hash=sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1 \
+    --hash=sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907 \
+    --hash=sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c \
+    --hash=sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3 \
+    --hash=sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b \
+    --hash=sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614 \
+    --hash=sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb \
+    --hash=sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b \
+    --hash=sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41 \
+    --hash=sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6 \
+    --hash=sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34 \
+    --hash=sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe \
+    --hash=sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4 \
+    --hash=sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7
new file mode 100644
--- /dev/null
+++ b/tools/lint/test/files/pylint/bad.py
@@ -0,0 +1,5 @@
+def foo():
+    useless_var = 1
+    useless_var = true
+    return "true"
+    print("unreachable")
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/tools/lint/test/files/pylint/good.py
@@ -0,0 +1,3 @@
+def foo():
+    a = 1 + 1
+    return a
--- a/tools/lint/test/python.ini
+++ b/tools/lint/test/python.ini
@@ -19,8 +19,12 @@ skip-if = os == "win" || os == "mac"  # 
 [test_yaml.py]
 skip-if = os == "win" || os == "mac"  # fails with No module named 'pkg_resources'
 [test_clippy.py]
 skip-if = os == "win" || os == "mac"  # only installed on Linux
 [test_rustfmt.py]
 skip-if = os == "win" || os == "mac"  # only installed on Linux
 [test_clang_format.py]
 skip-if = os == "win" || os == "mac"  # only installed on Linux
+[test_pylint.py]
+skip-if = os == "win" || os == "mac"  # only installed on linux
+requirements = tools/lint/python/pylint_requirements.txt
+
new file mode 100644
--- /dev/null
+++ b/tools/lint/test/test_pylint.py
@@ -0,0 +1,24 @@
+import mozunit
+
+LINTER = 'pylint'
+
+
+def test_lint_single_file(lint, paths):
+    results = lint(paths('bad.py'))
+    assert len(results) == 3
+    assert results[0].rule == 'E0602'
+    assert results[1].rule == 'W0101'
+    assert results[1].lineno == 5
+
+    # run lint again to make sure the previous results aren't counted twice
+    results = lint(paths('bad.py'))
+    assert len(results) == 3
+
+
+def test_lint_single_file_good(lint, paths):
+    results = lint(paths('good.py'))
+    assert len(results) == 0
+
+
+if __name__ == '__main__':
+    mozunit.main()