Bug 1391019 - Add py2 and py3 compatability linters, r=gps
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 31 Aug 2017 10:12:02 -0400
changeset 378032 ad881fc5e6b01f2c74d3a325b6c622a3ccb606c5
parent 378031 4c74a9958b31e99aef1e43b05efd7c48d057d595
child 378033 bb9206cf5ceb31a15dfd3c352a0dba140394a4af
push id50162
push userahalberstadt@mozilla.com
push dateThu, 31 Aug 2017 19:38:30 +0000
treeherderautoland@bb9206cf5ceb [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1391019
milestone57.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1391019 - Add py2 and py3 compatability linters, r=gps check_compat.py was adapted from gps' check-py3-compat.py in mercurial: https://www.mercurial-scm.org/repo/hg/file/tip/contrib/check-py3-compat.py The py3 linter simply runs ast.parse(f) for each file being linted. Any syntax errors are formatted as mozlint results and dumped to stdout as json. I looked into also importing the file (using 3.5+'s importlib.util.spec_from_file_location), but there were too many problems: 1. Lots of false positives (e.g module not found) 2. Some files seemed to run indefinitely on import I decided to punt on importing for now, we can always investigate in a follow-up. The py2 linter runs ast.parse(f), and also checks that the file has: from __future__ import absolute_import, print_function Initially every python file in the tree is excluded from the py2 check, though at least this makes it easy to find+fix, and new files in un-excluded directories will automatically be linted. MozReview-Commit-ID: ABtq9dnPo9T
tools/lint/py2.yml
tools/lint/py3.yml
tools/lint/python/check_compat.py
tools/lint/python/compat.py
new file mode 100644
--- /dev/null
+++ b/tools/lint/py2.yml
@@ -0,0 +1,75 @@
+---
+py2:
+    description: Python 2 compatibility check
+    include: ['.']
+    exclude:
+        - accessible/xpcom/AccEventGen.py
+        - addon-sdk
+        - browser
+        - build
+        - client.py
+        - config
+        - configure.py
+        - devtools/shared/css/generated/mach_commands.py
+        - dom
+        - editor
+        - gfx
+        - intl
+        - ipc
+        - js/src
+        - js/xpconnect
+        - layout
+        - media
+        - memory
+        - mobile
+        - modules
+        - mozglue
+        - netwerk
+        - nsprpub
+        - other-licenses
+        - probes/trace-gen.py
+        - python/devtools
+        - python/mach
+        - python/mozboot
+        - python/mozbuild
+        - python/mozlint
+        - python/mozversioncontrol
+        - security
+        - services/common/tests/mach_commands.py
+        - servo
+        - taskcluster/docker
+        - taskcluster/taskgraph
+        - testing/awsy
+        - testing/firefox-ui
+        - testing/geckodriver
+        - testing/gtest
+        - testing/instrumentation/runinstrumentation.py
+        - testing/marionette
+        - testing/mochitest
+        - testing/mozbase
+        - testing/mozharness
+        - testing/remotecppunittests.py
+        - testing/runcppunittests.py
+        - testing/runtimes
+        - testing/talos
+        - testing/tools
+        - testing/tps
+        - testing/web-platform
+        - testing/xpcshell
+        - third_party
+        - toolkit
+        - tools/docs
+        - tools/git/eslintvalidate.py
+        - tools/jprof/split-profile.py
+        - tools/lint
+        - tools/mach_commands.py
+        - tools/mercurial/eslintvalidate.py
+        - tools/power/mach_commands.py
+        - tools/profiler
+        - tools/rb
+        - tools/tryselect
+        - tools/update-packaging
+        - xpcom
+    extensions: ['py']
+    type: external
+    payload: python.compat:lintpy2
new file mode 100644
--- /dev/null
+++ b/tools/lint/py3.yml
@@ -0,0 +1,55 @@
+---
+py3:
+    description: Python 3 compatibility check
+    include: ['.']
+    exclude:
+        - addon-sdk/source
+        - browser/app
+        - browser/components
+        - browser/extensions
+        - build
+        - client.py
+        - config
+        - dom/bindings
+        - dom/canvas/test
+        - dom/media/test
+        - gfx
+        - intl/icu
+        - ipc/chromium
+        - ipc/ipdl
+        - js/src
+        - layout/reftests
+        - layout/style
+        - layout/tools/reftest
+        - media
+        - memory/replace
+        - modules/freetype2
+        - nsprpub
+        - security/manager/ssl
+        - security/nss
+        - services/common/tests/mach_commands.py
+        - servo
+        - testing/awsy
+        - testing/firefox-ui/harness/firefox_ui_harness/runners/update.py
+        - testing/gtest
+        - testing/marionette
+        - testing/mochitest
+        - testing/mozbase
+        - testing/mozharness
+        - testing/talos
+        - testing/tools/iceserver
+        - testing/tps
+        - testing/xpcshell
+        - testing/web-platform
+        - third_party
+        - toolkit
+        - tools/git
+        - tools/jprof
+        - tools/profiler
+        - tools/rb
+        - tools/update-packaging
+        - xpcom/idl-parser
+        - xpcom/typelib
+    extensions: ['py']
+    type: external
+    payload: python.compat:lintpy3
new file mode 100755
--- /dev/null
+++ b/tools/lint/python/check_compat.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import ast
+import json
+import sys
+
+
+def parse_file(f):
+    with open(f, 'rb') as fh:
+        content = fh.read()
+    try:
+        return ast.parse(content)
+    except SyntaxError as e:
+        err = {
+            'path': f,
+            'message': e.msg,
+            'lineno': e.lineno,
+            'column': e.offset,
+            'source': e.text,
+            'rule': 'is-parseable',
+        }
+        print(json.dumps(err))
+
+
+def check_compat_py2(f):
+    """Check Python 2 and Python 3 compatibility for a file with Python 2"""
+    root = parse_file(f)
+
+    # Ignore empty or un-parseable files.
+    if not root or not root.body:
+        return
+
+    futures = set()
+    haveprint = False
+    future_lineno = 1
+    for node in ast.walk(root):
+        if isinstance(node, ast.ImportFrom):
+            if node.module == '__future__':
+                future_lineno = node.lineno
+                futures |= set(n.name for n in node.names)
+        elif isinstance(node, ast.Print):
+            haveprint = True
+
+    err = {
+        'path': f,
+        'lineno': future_lineno,
+        'column': 1,
+    }
+
+    if 'absolute_import' not in futures:
+        err['rule'] = 'require absolute_import'
+        err['message'] = 'Missing from __future__ import absolute_import'
+        print(json.dumps(err))
+
+    if haveprint and 'print_function' not in futures:
+        err['rule'] = 'require print_function'
+        err['message'] = 'Missing from __future__ import print_function'
+        print(json.dumps(err))
+
+
+def check_compat_py3(f):
+    """Check Python 3 compatibility of a file with Python 3."""
+    parse_file(f)
+
+
+if __name__ == '__main__':
+    if sys.version_info[0] == 2:
+        fn = check_compat_py2
+    else:
+        fn = check_compat_py3
+
+    manifest = sys.argv[1]
+    with open(manifest, 'r') as fh:
+        files = fh.read().splitlines()
+
+    for f in files:
+        fn(f)
+
+    sys.exit(0)
new file mode 100644
--- /dev/null
+++ b/tools/lint/python/compat.py
@@ -0,0 +1,80 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import json
+import os
+import tempfile
+from distutils.spawn import find_executable
+
+from mozpack.files import FileFinder
+from mozprocess import ProcessHandlerMixin
+
+from mozlint import result
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+results = []
+
+
+class PyCompatProcess(ProcessHandlerMixin):
+    def __init__(self, config, *args, **kwargs):
+        self.config = config
+        kwargs['processOutputLine'] = [self.process_line]
+        ProcessHandlerMixin.__init__(self, *args, **kwargs)
+
+    def process_line(self, line):
+        try:
+            res = json.loads(line)
+        except ValueError:
+            print('Non JSON output from linter, will not be processed: {}'.format(line))
+            return
+
+        res['level'] = 'error'
+        results.append(result.from_config(self.config, **res))
+
+
+def run_linter(python, paths, config, **lintargs):
+    binary = find_executable(python)
+    if not binary:
+        # TODO bootstrap python3 if not available
+        print('error: {} not detected, aborting py-compat check'.format(python))
+        if 'MOZ_AUTOMATION' in os.environ:
+            return 1
+        return []
+
+    pattern = "**/*.py"
+    exclude = lintargs.get('exclude', [])
+    files = []
+    for path in paths:
+        if os.path.isfile(path):
+            files.append(path)
+            continue
+
+        finder = FileFinder(path, ignore=exclude)
+        files.extend([os.path.join(path, p) for p, f in finder.find(pattern)])
+
+    with tempfile.NamedTemporaryFile(mode='w') as fh:
+        fh.write('\n'.join(files))
+        fh.flush()
+
+        cmd = [binary, os.path.join(here, 'check_compat.py'), fh.name]
+
+        proc = PyCompatProcess(config, cmd)
+        proc.run()
+        try:
+            proc.wait()
+        except KeyboardInterrupt:
+            proc.kill()
+
+    return results
+
+
+def lintpy2(*args, **kwargs):
+    return run_linter('python2', *args, **kwargs)
+
+
+def lintpy3(*args, **kwargs):
+    return run_linter('python3', *args, **kwargs)