Bug 1498215 - Fix problem importing scandir when system scandir exists, r=davehunt
☠☠ backed out by 2099bd3c295d ☠ ☠
authorJames Graham <james@hoppipolla.co.uk>
Thu, 11 Oct 2018 15:05:16 +0000
changeset 499204 b01876f4f16eddbcfdc203e99f32356a81763863
parent 499203 7bd7b375eb2ec849a949ba93abd30b3ea6f97919
child 499205 ed70a55e2721cc9e84a05ecec3b67ef296f15950
push id1864
push userffxbld-merge
push dateMon, 03 Dec 2018 15:51:40 +0000
treeherdermozilla-release@f040763d99ad [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdavehunt
bugs1498215
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1498215 - Fix problem importing scandir when system scandir exists, r=davehunt If scandir is already present on the system the attempt to import the c helper library will currently find the c helper from the system install which may well be an outdated verion, so causing mach to break. To solve this this patch does two things: * Stops importing scandir in files that are run unconditionally when invoking mach. This is generally considered good for performance reasons. * Installs the vendored scandir into the virtualenv for `mach lint` rather than trying to import it directly from the source tree and so not getting the c helper library. Differential Revision: https://phabricator.services.mozilla.com/D8379
build/virtualenv_packages.txt
python/mozlint/mozlint/pathutils.py
tools/lint/mach_commands.py
--- a/build/virtualenv_packages.txt
+++ b/build/virtualenv_packages.txt
@@ -26,17 +26,16 @@ mozilla.pth:third_party/python/jsmin
 !windows:mozilla.pth:third_party/python/psutil
 windows:mozilla.pth:third_party/python/psutil-cp27-none-win_amd64
 mozilla.pth:third_party/python/pylru
 mozilla.pth:third_party/python/which
 mozilla.pth:third_party/python/pystache
 mozilla.pth:third_party/python/pyyaml/lib
 mozilla.pth:third_party/python/requests
 mozilla.pth:third_party/python/requests-unixsocket
-mozilla.pth:third_party/python/scandir
 mozilla.pth:third_party/python/slugid
 mozilla.pth:third_party/python/py
 mozilla.pth:third_party/python/pytest/src
 mozilla.pth:third_party/python/pytoml
 mozilla.pth:third_party/python/redo
 mozilla.pth:third_party/python/six
 mozilla.pth:third_party/python/voluptuous
 mozilla.pth:third_party/python/json-e
--- a/python/mozlint/mozlint/pathutils.py
+++ b/python/mozlint/mozlint/pathutils.py
@@ -4,20 +4,23 @@
 
 from __future__ import unicode_literals, absolute_import
 
 import os
 
 from mozpack import path as mozpath
 from mozpack.files import FileFinder
 
-try:
-    from os import scandir, walk
-except ImportError:
-    from scandir import scandir, walk
+
+def get_scandir_functions():
+    try:
+        from os import scandir, walk
+    except ImportError:
+        from scandir import scandir, walk
+    return scandir, walk
 
 
 class FilterPath(object):
     """Helper class to make comparing and matching file paths easier."""
     def __init__(self, path):
         self.path = os.path.normpath(path)
         self._finder = None
 
@@ -82,16 +85,19 @@ def collapse(paths, base=None, dotfiles=
 
     returns ['a']. But if a third file d.txt also exists, then it will return
     ['a/b.txt', 'a/c.txt'] since ['a'] would also include that extra file.
 
     :param paths: An iterable of paths (files and directories) to collapse.
     :returns: The smallest set of paths (files and directories) that contain
               the original set of paths and only the original set.
     """
+    # Import scandir locally because this file can be imported before we
+    # have created a correctly configured environment
+    scandir, walk = get_scandir_functions()
     if not paths:
         if not base:
             return []
 
         # Need to test whether directory chain is empty. If it is then bubble
         # the base back up so that it counts as 'covered'.
         for _, _, names in walk(base):
             if names:
--- a/tools/lint/mach_commands.py
+++ b/tools/lint/mach_commands.py
@@ -13,50 +13,64 @@ from mozbuild.base import (
 
 
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
 )
 
-try:
-    from os import scandir
-except ImportError:
-    from scandir import scandir
-
-
 here = os.path.abspath(os.path.dirname(__file__))
 GLOBAL_EXCLUDES = [
     'tools/lint/test/files',
 ]
 
 
 def setup_argument_parser():
     from mozlint import cli
     return cli.MozlintParser()
 
 
 def get_global_excludes(topsrcdir):
+    try:
+        from os import scandir
+    except ImportError:
+        from scandir import scandir
+
     excludes = GLOBAL_EXCLUDES[:]
     excludes.extend([e.name for e in scandir(topsrcdir)
                      if e.name.startswith('obj') and e.is_dir()])
     return excludes
 
 
 @CommandProvider
 class MachCommands(MachCommandBase):
 
     @Command(
         'lint', category='devenv',
         description='Run linters.',
         parser=setup_argument_parser)
     def lint(self, *runargs, **lintargs):
         """Run linters."""
         self._activate_virtualenv()
+        try:
+            from os import scandir  # noqa
+        except ImportError:
+            # Install scandir from the source tree into the virtualenv
+            # This is different to the way that most packages are installed,
+            # to ensure that we get the C helper built where possible; the normal
+            # approach of a pointer into the source directory doesn't work for this
+            # case.
+            self.virtualenv_manager.install_pip_package(
+                os.path.join(here,
+                             os.path.pardir,
+                             os.path.pardir,
+                             'third_party',
+                             'python',
+                             'scandir'), vendored=True)
         from mozlint import cli
 
         lintargs.setdefault('root', self.topsrcdir)
         lintargs['exclude'] = get_global_excludes(lintargs['root'])
         cli.SEARCH_PATHS.append(here)
         return cli.run(*runargs, **lintargs)
 
     @Command('eslint', category='devenv',