Bug 1288225 - Exclude eslint-plugin-mozilla from tooltool. r=ahal
authorJ. Ryan Stinnett <jryans@gmail.com>
Fri, 22 Jul 2016 18:18:58 -0500
changeset 347179 410cc90a90da1146d73bb27b3e3018a5be95db6a
parent 347178 d865632280ba4212731114d90babce298d79ee6e
child 347180 489c6ca83f4910adb9679832521d13fa28febc98
push id6389
push userraliiev@mozilla.com
push dateMon, 19 Sep 2016 13:38:22 +0000
treeherdermozilla-beta@01d67bfe6c81 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1288225
milestone50.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1288225 - Exclude eslint-plugin-mozilla from tooltool. r=ahal This removes the in-tree plugin from the tooltool archive and uses that code directly from the Gecko checkout instead. For automation, we now get ESLint and external plugins from tooltool and then symbolic link to the in-tree plugin. For local development, we install ESLint and external plugins following the shrinkwrap file created from the last change to the tooltool archive. The local plugin is then installed. This change also removes the list of module versions from mach_commands.py, so there is only one place to update module versions for the future. MozReview-Commit-ID: AhbZ8lVPmN4
taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
tools/lint/eslint/manifest.tt
tools/lint/eslint/npm-shrinkwrap.json
tools/lint/eslint/package.json
tools/lint/eslint/update
tools/lint/mach_commands.py
--- a/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
+++ b/taskcluster/ci/legacy/tasks/tests/eslint-gecko.yml
@@ -24,16 +24,17 @@ task:
       - /home/worker/workspace/gecko
       - bash
       - -cx
       - >
           cd /home/worker/workspace/gecko/tools/lint/eslint &&
           /build/tooltool.py fetch -m manifest.tt &&
           tar xvfz eslint.tar.gz &&
           rm eslint.tar.gz &&
+          ln -s ../eslint-plugin-mozilla node_modules &&
           cd ../../.. &&
           tools/lint/eslint/node_modules/.bin/eslint --quiet --plugin html --ext [.js,.jsm,.jsx,.xml,.html] -f tools/lint/eslint-formatter .
 
   extra:
     locations:
         build: null
         tests: null
     treeherder:
--- a/tools/lint/eslint/manifest.tt
+++ b/tools/lint/eslint/manifest.tt
@@ -1,9 +1,9 @@
 [
 {
-"size": 2349680,
+"size": 2371391,
 "visibility": "public",
-"digest": "2b02ae6dd4bc735990660f97a831f05e604c28120977e4120cf59619fb02be22cbd42be26ec2bd176f172f4566f3dfb445082e8d9651346662b8fb8fde407b8c",
+"digest": "31421d2c6dbf6d8ab6ee45f29b191d1932b7f84a5f8a9cdf24d071a39be75c7013c7db07d79921cbb300678820bd0abca600edbd0b51bf907f0a7eafd3f91382",
 "algorithm": "sha512",
 "filename": "eslint.tar.gz"
 }
 ]
--- a/tools/lint/eslint/npm-shrinkwrap.json
+++ b/tools/lint/eslint/npm-shrinkwrap.json
@@ -1,14 +1,15 @@
 {
+  "name": "mach-eslint",
   "dependencies": {
     "acorn": {
-      "version": "3.2.0",
+      "version": "3.3.0",
       "from": "acorn@>=3.1.0 <4.0.0",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.2.0.tgz"
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz"
     },
     "acorn-jsx": {
       "version": "3.0.1",
       "from": "acorn-jsx@>=3.0.0 <4.0.0",
       "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-3.0.1.tgz"
     },
     "ansi-escapes": {
       "version": "1.4.0",
@@ -201,71 +202,61 @@
     },
     "escape-string-regexp": {
       "version": "1.0.5",
       "from": "escape-string-regexp@>=1.0.2 <2.0.0",
       "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz"
     },
     "escope": {
       "version": "3.6.0",
-      "from": "escope@>=3.6.0 <4.0.0",
+      "from": "escope@>=3.2.0 <4.0.0",
       "resolved": "https://registry.npmjs.org/escope/-/escope-3.6.0.tgz"
     },
     "eslint": {
       "version": "2.9.0",
       "from": "eslint@2.9.0",
-      "resolved": "https://registry.npmjs.org/eslint/-/eslint-2.9.0.tgz"
+      "resolved": "https://registry.npmjs.org/eslint/-/eslint-2.9.0.tgz",
+      "dependencies": {
+        "espree": {
+          "version": "3.1.4",
+          "from": "espree@3.1.4",
+          "resolved": "https://registry.npmjs.org/espree/-/espree-3.1.4.tgz"
+        }
+      }
     },
     "eslint-plugin-html": {
       "version": "1.4.0",
       "from": "eslint-plugin-html@1.4.0",
       "resolved": "https://registry.npmjs.org/eslint-plugin-html/-/eslint-plugin-html-1.4.0.tgz"
     },
-    "eslint-plugin-mozilla": {
-      "version": "0.2.0",
-      "from": "eslint-plugin-mozilla",
-      "resolved": "file:eslint-plugin-mozilla",
-      "dependencies": {
-        "espree": {
-          "version": "2.2.5",
-          "from": "espree@>=2.2.4 <3.0.0",
-          "resolved": "https://registry.npmjs.org/espree/-/espree-2.2.5.tgz"
-        }
-      }
-    },
     "eslint-plugin-react": {
       "version": "4.2.3",
       "from": "eslint-plugin-react@4.2.3",
       "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-4.2.3.tgz"
     },
     "espree": {
-      "version": "3.1.4",
-      "from": "espree@3.1.4",
-      "resolved": "https://registry.npmjs.org/espree/-/espree-3.1.4.tgz"
-    },
-    "esprima": {
-      "version": "2.7.2",
-      "from": "esprima@>=2.6.0 <3.0.0",
-      "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.2.tgz"
+      "version": "2.2.5",
+      "from": "espree@>=2.2.4 <3.0.0",
+      "resolved": "https://registry.npmjs.org/espree/-/espree-2.2.5.tgz"
     },
     "esrecurse": {
       "version": "4.1.0",
       "from": "esrecurse@>=4.1.0 <5.0.0",
       "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.1.0.tgz",
       "dependencies": {
         "estraverse": {
           "version": "4.1.1",
           "from": "estraverse@>=4.1.0 <4.2.0",
           "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.1.1.tgz"
         }
       }
     },
     "estraverse": {
       "version": "4.2.0",
-      "from": "estraverse@>=4.2.0 <5.0.0",
+      "from": "estraverse@>=4.1.1 <5.0.0",
       "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz"
     },
     "esutils": {
       "version": "2.0.2",
       "from": "esutils@>=2.0.2 <3.0.0",
       "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz"
     },
     "event-emitter": {
@@ -324,19 +315,19 @@
       "resolved": "https://registry.npmjs.org/globals/-/globals-9.9.0.tgz"
     },
     "globby": {
       "version": "5.0.0",
       "from": "globby@>=5.0.0 <6.0.0",
       "resolved": "https://registry.npmjs.org/globby/-/globby-5.0.0.tgz"
     },
     "graceful-fs": {
-      "version": "4.1.4",
+      "version": "4.1.5",
       "from": "graceful-fs@>=4.1.2 <5.0.0",
-      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.4.tgz"
+      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.5.tgz"
     },
     "has-ansi": {
       "version": "2.0.0",
       "from": "has-ansi@>=2.0.0 <3.0.0",
       "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz"
     },
     "htmlparser2": {
       "version": "3.9.1",
@@ -406,17 +397,24 @@
     "isarray": {
       "version": "1.0.0",
       "from": "isarray@>=1.0.0 <1.1.0",
       "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz"
     },
     "js-yaml": {
       "version": "3.6.1",
       "from": "js-yaml@>=3.5.1 <4.0.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.6.1.tgz"
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.6.1.tgz",
+      "dependencies": {
+        "esprima": {
+          "version": "2.7.2",
+          "from": "esprima@>=2.6.0 <3.0.0",
+          "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.2.tgz"
+        }
+      }
     },
     "json-stable-stringify": {
       "version": "1.0.1",
       "from": "json-stable-stringify@>=1.0.0 <2.0.0",
       "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz"
     },
     "jsonify": {
       "version": "0.0.0",
@@ -429,19 +427,19 @@
       "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-2.0.0.tgz"
     },
     "levn": {
       "version": "0.3.0",
       "from": "levn@>=0.3.0 <0.4.0",
       "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz"
     },
     "lodash": {
-      "version": "4.13.1",
+      "version": "4.14.0",
       "from": "lodash@>=4.0.0 <5.0.0",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.13.1.tgz"
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.14.0.tgz"
     },
     "minimatch": {
       "version": "3.0.2",
       "from": "minimatch@>=3.0.2 <4.0.0",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.2.tgz"
     },
     "minimist": {
       "version": "0.0.8",
--- a/tools/lint/eslint/package.json
+++ b/tools/lint/eslint/package.json
@@ -1,12 +1,15 @@
 {
-  "name": "",
-  "description": "None",
+  "name": "mach-eslint",
+  "description": "ESLint and external plugins for use with mach",
   "repository": {},
   "license": "MPL-2.0",
   "dependencies": {
-    "eslint": "*",
-    "eslint-plugin-html": "*",
-    "eslint-plugin-mozilla": "*",
-    "eslint-plugin-react": "*"
+    "eslint": "2.9.0",
+    "eslint-plugin-html": "1.4.0",
+    "eslint-plugin-react": "4.2.3",
+    "escope": "^3.2.0",
+    "espree": "^2.2.4",
+    "estraverse": "^4.1.1",
+    "sax": "^1.1.4"
   }
 }
--- a/tools/lint/eslint/update
+++ b/tools/lint/eslint/update
@@ -32,18 +32,23 @@ case "$choice" in
     ;;
 esac
 
 echo ""
 echo "Removing node_modules and npm_shrinkwrap.json..."
 rm -rf node_modules/
 rm npm-shrinkwrap.json
 
-echo "Installing eslint and dependencies..."
-../../../mach eslint --setup
+echo "Installing eslint and external plugins..."
+# ESLint and all _external_ plugins are listed in this directory's package.json,
+# so a regular `npm install` will install them at the specified versions.
+# The in-tree eslint-plugin-mozilla is kept out of this tooltool archive on
+# purpose so that it can be changed by any developer without requiring tooltool
+# access to make changes.
+npm install
 
 echo "Creating npm shrinkwrap..."
 npm shrinkwrap
 
 echo "Creating eslint.tar.gz..."
 tar cvfz eslint.tar.gz node_modules
 
 echo "Downloading tooltool..."
--- a/tools/lint/mach_commands.py
+++ b/tools/lint/mach_commands.py
@@ -4,16 +4,17 @@
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
 import json
 import logging
 import os
 import platform
+import re
 import subprocess
 import sys
 import which
 from distutils.version import LooseVersion
 
 from mozbuild.base import (
     MachCommandBase,
 )
@@ -24,23 +25,16 @@ from mach.decorators import (
     CommandProvider,
     Command,
 )
 
 
 here = os.path.abspath(os.path.dirname(__file__))
 
 
-ESLINT_PACKAGES = [
-    "eslint@2.9.0",
-    "eslint-plugin-html@1.4.0",
-    "eslint-plugin-mozilla@0.2.0",
-    "eslint-plugin-react@4.2.3"
-]
-
 ESLINT_NOT_FOUND_MESSAGE = '''
 Could not find eslint!  We looked at the --binary option, at the ESLINT
 environment variable, and then at your local node_modules path. Please Install
 eslint and needed plugins with:
 
 mach eslint --setup
 
 and try again.
@@ -56,16 +50,19 @@ Valid installation paths:
 NPM_NOT_FOUND_MESSAGE = '''
 Node Package Manager (npm) is either not installed or installed to a
 non-standard path. Please install npm from https://nodejs.org (it comes as an
 option in the node installation) and try again.
 
 Valid installation paths:
 '''.strip()
 
+VERSION_RE = re.compile(r"^\d+\.\d+\.\d+$")
+CARET_VERSION_RANGE_RE = re.compile(r"^\^((\d+)\.\d+\.\d+)$")
+
 
 def setup_argument_parser():
     from mozlint import cli
     return cli.MozlintParser()
 
 
 @CommandProvider
 class MachCommands(MachCommandBase):
@@ -79,17 +76,17 @@ class MachCommands(MachCommandBase):
         from mozlint import cli
         lintargs['exclude'] = ['obj*']
         cli.SEARCH_PATHS.append(here)
         return cli.run(*runargs, **lintargs)
 
     @Command('eslint', category='devenv',
              description='Run eslint or help configure eslint for optimal development.')
     @CommandArgument('-s', '--setup', default=False, action='store_true',
-                     help='configure eslint for optimal development.')
+                     help='Configure eslint for optimal development.')
     @CommandArgument('-e', '--ext', default='[.js,.jsm,.jsx,.xml,.html]',
                      help='Filename extensions to lint, default: "[.js,.jsm,.jsx,.xml,.html]".')
     @CommandArgument('-b', '--binary', default=None,
                      help='Path to eslint binary.')
     @CommandArgument('args', nargs=argparse.REMAINDER)  # Passed through to eslint.
     def eslint(self, setup, ext=None, binary=None, args=None):
         '''Run eslint.'''
 
@@ -98,18 +95,18 @@ class MachCommands(MachCommandBase):
         # eslint requires at least node 4.2.3
         nodePath = self.get_node_or_npm_path("node", LooseVersion("4.2.3"))
         if not nodePath:
             return 1
 
         if setup:
             return self.eslint_setup()
 
-        npmPath = self.get_node_or_npm_path("npm")
-        if not npmPath:
+        npm_path = self.get_node_or_npm_path("npm")
+        if not npm_path:
             return 1
 
         if self.eslint_module_has_issues():
             install = self._prompt_yn("\nContinuing will automatically fix "
                                       "these issues. Would you like to "
                                       "continue")
             if install:
                 self.eslint_setup()
@@ -157,56 +154,48 @@ class MachCommands(MachCommandBase):
             ensure_exit_code=False,  # Don't throw on non-zero exit code.
             require_unix_environment=True  # eslint is not a valid Win32 binary.
         )
 
         self.log(logging.INFO, 'eslint', {'msg': ('No errors' if success == 0 else 'Errors')},
                  'Finished eslint. {msg} encountered.')
         return success
 
-    def eslint_setup(self, update_only=False):
+    def eslint_setup(self):
         """Ensure eslint is optimally configured.
 
         This command will inspect your eslint configuration and
         guide you through an interactive wizard helping you configure
         eslint for optimal use on Mozilla projects.
         """
         orig_cwd = os.getcwd()
         sys.path.append(os.path.dirname(__file__))
 
         module_path = self.get_eslint_module_path()
 
         # npm sometimes fails to respect cwd when it is run using check_call so
         # we manually switch folders here instead.
         os.chdir(module_path)
 
-        npmPath = self.get_node_or_npm_path("npm")
-        if not npmPath:
+        npm_path = self.get_node_or_npm_path("npm")
+        if not npm_path:
             return 1
 
-        # Install eslint and necessary plugins.
-        for pkg in ESLINT_PACKAGES:
-            name, version = pkg.split("@")
-            success = False
+        # Install ESLint and external plugins
+        cmd = [npm_path, "install"]
+        print("Installing eslint for mach using \"%s\"..." % (" ".join(cmd)))
+        if not self.call_process("eslint", cmd):
+            return 1
 
-            if self.node_package_installed(pkg, cwd=module_path):
-                success = True
-            else:
-                if pkg.startswith("eslint-plugin-mozilla"):
-                    cmd = [npmPath, "install",
-                           os.path.join(module_path, "eslint-plugin-mozilla")]
-                else:
-                    cmd = [npmPath, "install", pkg]
-
-                print("Installing %s v%s using \"%s\"..."
-                      % (name, version, " ".join(cmd)))
-                success = self.call_process(pkg, cmd)
-
-            if not success:
-                return 1
+        # Install in-tree ESLint plugin
+        cmd = [npm_path, "install",
+               os.path.join(module_path, "eslint-plugin-mozilla")]
+        print("Installing eslint-plugin-mozilla using \"%s\"..." % (" ".join(cmd)))
+        if not self.call_process("eslint-plugin-mozilla", cmd):
+            return 1
 
         eslint_path = os.path.join(module_path, "node_modules", ".bin", "eslint")
 
         print("\nESLint and approved plugins installed successfully!")
         print("\nNOTE: Your local eslint binary is at %s\n" % eslint_path)
 
         os.chdir(orig_cwd)
 
@@ -219,52 +208,77 @@ class MachCommands(MachCommandBase):
                 print("\nError installing %s in the %s folder, aborting." % (name, cwd))
             else:
                 print("\nError installing %s, aborting." % name)
 
             return False
 
         return True
 
+    def expected_eslint_modules(self):
+        # Read the expected version of ESLint and external modules
+        expected_modules_path = os.path.join(self.get_eslint_module_path(), "package.json")
+        with open(expected_modules_path, "r") as f:
+            expected_modules = json.load(f)["dependencies"]
+
+        # Also read the in-tree ESLint plugin version
+        mozilla_json_path = os.path.join(self.get_eslint_module_path(),
+                                         "eslint-plugin-mozilla", "package.json")
+        with open(mozilla_json_path, "r") as f:
+            expected_modules["eslint-plugin-mozilla"] = json.load(f)["version"]
+
+        return expected_modules
+
     def eslint_module_has_issues(self):
         has_issues = False
-        node_module_path = os.path.join(self.get_eslint_module_path(), "node_modules")
+        node_modules_path = os.path.join(self.get_eslint_module_path(), "node_modules")
 
-        for pkg in ESLINT_PACKAGES:
-            name, req_version = pkg.split("@")
-            path = os.path.join(node_module_path, name, "package.json")
+        for name, version_range in self.expected_eslint_modules().iteritems():
+            path = os.path.join(node_modules_path, name, "package.json")
 
             if not os.path.exists(path):
-                print("%s v%s needs to be installed locally." % (name, req_version))
+                print("%s v%s needs to be installed locally." % (name, version_range))
                 has_issues = True
                 continue
 
             data = json.load(open(path))
 
-            if data["version"] != req_version:
-                print("%s v%s should be v%s." % (name, data["version"], req_version))
+            if not self.version_in_range(data["version"], version_range):
+                print("%s v%s should be v%s." % (name, data["version"], version_range))
                 has_issues = True
 
         return has_issues
 
-    def node_package_installed(self, package_name="", globalInstall=False, cwd=None):
-        try:
-            npmPath = self.get_node_or_npm_path("npm")
+    def version_in_range(self, version, version_range):
+        """
+        Check if a module version is inside a version range.  Only supports explicit versions and
+        caret ranges for the moment, since that's all we've used so far.
+        """
+        if version == version_range:
+            return True
 
-            cmd = [npmPath, "ls", "--parseable", package_name]
+        version_match = VERSION_RE.match(version)
+        if not version_match:
+            raise RuntimeError("mach eslint doesn't understand module version %s" % version)
+        version = LooseVersion(version)
 
-            if globalInstall:
-                cmd.append("-g")
+        # Caret ranges as specified by npm allow changes that do not modify the left-most non-zero
+        # digit in the [major, minor, patch] tuple.  The code below assumes the major digit is
+        # non-zero.
+        range_match = CARET_VERSION_RANGE_RE.match(version_range)
+        if range_match:
+            range_version = range_match.group(1)
+            range_major = int(range_match.group(2))
 
-            with open(os.devnull, "w") as fnull:
-                subprocess.check_call(cmd, stdout=fnull, stderr=fnull, cwd=cwd)
+            range_min = LooseVersion(range_version)
+            range_max = LooseVersion("%d.0.0" % (range_major + 1))
 
-            return True
-        except subprocess.CalledProcessError:
-            return False
+            return range_min <= version < range_max
+
+        return False
 
     def get_possible_node_paths_win(self):
         """
         Return possible nodejs paths on Windows.
         """
         if platform.system() != "Windows":
             return []
 
@@ -277,27 +291,27 @@ class MachCommands(MachCommandBase):
 
     def get_node_or_npm_path(self, filename, minversion=None):
         """
         Return the nodejs or npm path.
         """
         if platform.system() == "Windows":
             for ext in [".cmd", ".exe", ""]:
                 try:
-                    nodeOrNpmPath = which.which(filename + ext,
-                                                path=self.get_possible_node_paths_win())
-                    if self.is_valid(nodeOrNpmPath, minversion):
-                        return nodeOrNpmPath
+                    node_or_npm_path = which.which(filename + ext,
+                                                   path=self.get_possible_node_paths_win())
+                    if self.is_valid(node_or_npm_path, minversion):
+                        return node_or_npm_path
                 except which.WhichError:
                     pass
         else:
             try:
-                nodeOrNpmPath = which.which(filename)
-                if self.is_valid(nodeOrNpmPath, minversion):
-                    return nodeOrNpmPath
+                node_or_npm_path = which.which(filename)
+                if self.is_valid(node_or_npm_path, minversion):
+                    return node_or_npm_path
             except which.WhichError:
                 pass
 
         if filename == "node":
             print(NODE_NOT_FOUND_MESSAGE)
         elif filename == "npm":
             print(NPM_NOT_FOUND_MESSAGE)