Bug 1188224 - Remove stale .pyc files from the source directory at import time. r=gps
☠☠ backed out by 125b87d65303 ☠ ☠
authorMike Hommey <mh+mozilla@glandium.org>
Tue, 04 Aug 2015 16:59:21 +0900
changeset 287915 684252f11061afa8532693516a14730f41b56841
parent 287914 638db7656bee5286a3d385b1026acc770e7d4d42
child 287916 c0a6f89391ee20c27523d228a765ff22ce2b28fd
push id5067
push userraliiev@mozilla.com
push dateMon, 21 Sep 2015 14:04:52 +0000
treeherdermozilla-beta@14221ffe5b2f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1188224
milestone42.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1188224 - Remove stale .pyc files from the source directory at import time. r=gps
build/mach_bootstrap.py
python/mozbuild/mozbuild/virtualenv.py
--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -4,16 +4,17 @@
 
 from __future__ import print_function, unicode_literals
 
 import errno
 import os
 import platform
 import sys
 import time
+import __builtin__
 
 
 STATE_DIR_FIRST_RUN = '''
 mach and the build system store shared state in a common directory on the
 filesystem. The following directory will be created:
 
   {userdir}
 
@@ -322,8 +323,70 @@ def bootstrap(topsrcdir, mozilla_dir=Non
     for category, meta in CATEGORIES.items():
         mach.define_category(category, meta['short'], meta['long'],
             meta['priority'])
 
     for path in MACH_MODULES:
         mach.load_commands_from_file(os.path.join(mozilla_dir, path))
 
     return mach
+
+
+# Hook import such that .pyc/.pyo files without a corresponding .py file in
+# the source directory are essentially ignored. See further below for details
+# and caveats.
+# Objdirs outside the source directory are ignored because in most cases, if
+# a .pyc/.pyo file exists there, a .py file will be next to it anyways.
+class ImportHook(object):
+    def __init__(self, original_import):
+        self._original_import = original_import
+        # Assume the source directory is the parent directory of the one
+        # containing this file.
+        self._source_dir = os.path.normcase(os.path.abspath(
+            os.path.dirname(os.path.dirname(__file__)))) + os.sep
+        self._modules = set()
+
+    def __call__(self, name, globals=None, locals=None, fromlist=None,
+                 level=-1):
+        # name might be a relative import. Instead of figuring out what that
+        # resolves to, which is complex, just rely on the real import.
+        # Since we don't know the full module name, we can't check sys.modules,
+        # so we need to keep track of which modules we've already seen to avoid
+        # to stat() them again when they are imported multiple times.
+        module = self._original_import(name, globals, locals, fromlist, level)
+        resolved_name = module.__name__
+        if resolved_name in self._modules:
+            return module
+        self._modules.add(resolved_name)
+
+        # Builtin modules don't have a __file__ attribute.
+        if not hasattr(module, '__file__'):
+            return module
+
+        # Note: module.__file__ is not always absolute.
+        path = os.path.normcase(os.path.abspath(module.__file__))
+        # Note: we could avoid normcase and abspath above for non pyc/pyo
+        # files, but those are actually rare, so it doesn't really matter.
+        if not path.endswith(('.pyc', '.pyo')):
+            return module
+
+        # Ignore modules outside our source directory
+        if not path.startswith(self._source_dir):
+            return module
+
+        # If there is no .py corresponding to the .pyc/.pyo module we're
+        # loading, remove the .pyc/.pyo file, and reload the module.
+        # Since we already loaded the .pyc/.pyo module, if it had side
+        # effects, they will have happened already, and loading the module
+        # with the same name, from another directory may have the same side
+        # effects (or different ones). We assume it's not a problem for the
+        # python modules under our source directory (either because it
+        # doesn't happen or because it doesn't matter).
+        if not os.path.exists(module.__file__[:-1]):
+            os.remove(module.__file__)
+            del sys.modules[module.__name__]
+            module = self(name, globals, locals, fromlist, level)
+
+        return module
+
+
+# Install our hook
+__builtin__.__import__ = ImportHook(__builtin__.__import__)
--- a/python/mozbuild/mozbuild/virtualenv.py
+++ b/python/mozbuild/mozbuild/virtualenv.py
@@ -189,19 +189,19 @@ class VirtualenvManager(object):
 
         Note that the Python interpreter running this function should be the
         one from the virtualenv. If it is the system Python or if the
         environment is not configured properly, packages could be installed
         into the wrong place. This is how virtualenv's work.
         """
 
         packages = self.packages()
+        python_lib = distutils.sysconfig.get_python_lib()
 
         def handle_package(package):
-            python_lib = distutils.sysconfig.get_python_lib()
             if package[0] == 'setup.py':
                 assert len(package) >= 2
 
                 self.call_setup(os.path.join(self.topsrcdir, package[1]),
                     package[2:])
 
                 return True
 
@@ -326,16 +326,26 @@ class VirtualenvManager(object):
                 for ver in ('100', '110', '120'):
                     var = 'VS%sCOMNTOOLS' % ver
                     if var in os.environ:
                         os.environ['VS90COMNTOOLS'] = os.environ[var]
                         break
 
             for package in packages:
                 handle_package(package)
+
+            sitecustomize = os.path.join(
+                os.path.dirname(os.__file__), 'sitecustomize.py')
+            with open(sitecustomize, 'w') as f:
+                f.write(
+                    '# Importing mach_bootstrap has the side effect of\n'
+                    '# installing an import hook\n'
+                    'import mach_bootstrap\n'
+                )
+
         finally:
             os.environ.pop('MACOSX_DEPLOYMENT_TARGET', None)
 
             if old_target is not None:
                 os.environ['MACOSX_DEPLOYMENT_TARGET'] = old_target
 
             os.environ.update(old_env_variables)