Bug 1255585 - Prevent Python executable mis-match from constantly clobbering the virtualenv on OS X. r=gps
☠☠ backed out by 49c270ffc682 ☠ ☠
authorChris Manchester <cmanchester@mozilla.com>
Fri, 11 Mar 2016 12:24:10 -0800
changeset 288395 2ed1a0e47b18714f6d3af196d58250822f84208b
parent 288394 c8d3ebfc4d36b4d38069d0170e3caf36f7fd86fd
child 288396 025deb1d01f3ac4433cffad959f51a6f3f64ea24
push id30079
push userryanvm@gmail.com
push dateSat, 12 Mar 2016 20:24:19 +0000
treeherdermozilla-central@d1d47ba19ce9 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1255585
milestone48.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1255585 - Prevent Python executable mis-match from constantly clobbering the virtualenv on OS X. r=gps Virtualenv will sometimes find a different executable from its sys.executable on OS X, causing a check in the build system comparing filesizes between sys.executable and virtualenv python to fail, resulting in clobbering and re-building the virtualenv every time the virtualenv is activated, causing the build backend and more to be re-built. Instead of checking file sizes directly, this commit causes us to record the size and path to the Python executable that created the virtualenv. If the Python executable checked is not the virtualenv Python, or the python that was used to create the interpreter, then the virtualenv is considered to be out of date. MozReview-Commit-ID: KmrVfQCtbS3
python/mozbuild/mozbuild/virtualenv.py
--- a/python/mozbuild/mozbuild/virtualenv.py
+++ b/python/mozbuild/mozbuild/virtualenv.py
@@ -45,16 +45,23 @@ class VirtualenvManager(object):
 
         Each manager is associated with a source directory, a path where you
         want the virtualenv to be created, and a handle to write output to.
         """
         assert os.path.isabs(manifest_path), "manifest_path must be an absolute path: %s" % (manifest_path)
         self.topsrcdir = topsrcdir
         self.topobjdir = topobjdir
         self.virtualenv_root = virtualenv_path
+
+        # Record the Python executable that was used to create the Virtualenv
+        # so we can check this against sys.executable when verifying the
+        # integrity of the virtualenv.
+        self.exe_info_path = os.path.join(self.virtualenv_root,
+                                          'python_exe.txt')
+
         self.log_handle = log_handle
         self.manifest_path = manifest_path
 
     @property
     def virtualenv_script_path(self):
         """Path to virtualenv's own populator script."""
         return os.path.join(self.topsrcdir, 'python', 'virtualenv',
             'virtualenv.py')
@@ -77,16 +84,34 @@ class VirtualenvManager(object):
             binary += '.exe'
 
         return os.path.join(self.bin_path, binary)
 
     @property
     def activate_path(self):
         return os.path.join(self.bin_path, 'activate_this.py')
 
+    def get_exe_info(self):
+        """Returns the path to python executable that was in use when
+        this virutalenv was created.
+        """
+        with open(self.exe_info_path, 'r') as fh:
+            path, size = fh.read().splitlines()
+        return path, int(size)
+
+    def write_exe_info(self, python):
+        """Records the path to python executable that was in use when
+        this virutalenv was created. We record this explicitly because
+        on OS X our python path may end up being a different or modified
+        executable.
+        """
+        with open(self.exe_info_path, 'w') as fh:
+            fh.write("%s\n" % python)
+            fh.write("%s\n" % os.path.getsize(python))
+
     def up_to_date(self, python=sys.executable):
         """Returns whether the virtualenv is present and up to date."""
 
         deps = [self.manifest_path, __file__]
 
         # check if virtualenv exists
         if not os.path.exists(self.virtualenv_root) or \
             not os.path.exists(self.activate_path):
@@ -94,20 +119,23 @@ class VirtualenvManager(object):
             return False
 
         # check modification times
         activate_mtime = os.path.getmtime(self.activate_path)
         dep_mtime = max(os.path.getmtime(p) for p in deps)
         if dep_mtime > activate_mtime:
             return False
 
-        # check interpreter. Assume that a different size is enough to
-        # know whether we've been given a different interpreter, and thus
-        # should refresh the virtualenv.
-        if os.path.getsize(python) != os.path.getsize(self.python_path):
+        # Verify that the Python we're checking here is either the virutalenv
+        # python, or is the Python that was used to create the virtualenv.
+        # If this fails, it is likely system Python has been upgraded, and
+        # our virtualenv would not be usable.
+        python_size = os.path.getsize(python)
+        if ((python, python_size) != (self.python_path, os.path.getsize(self.python_path)) and
+            (python, python_size) != self.get_exe_info()):
             return False
 
         # recursively check sub packages.txt files
         submanifests = [i[1] for i in self.packages()
                         if i[0] == 'packages.txt']
         for submanifest in submanifests:
             submanifest = os.path.join(self.topsrcdir, submanifest)
             submanager = VirtualenvManager(self.topsrcdir,
@@ -148,16 +176,18 @@ class VirtualenvManager(object):
 
         result = subprocess.call(args, stdout=self.log_handle,
             stderr=subprocess.STDOUT, env=env)
 
         if result:
             raise Exception(
                 'Failed to create virtualenv: %s' % self.virtualenv_root)
 
+        self.write_exe_info(python)
+
         return self.virtualenv_root
 
     def packages(self):
         with file(self.manifest_path, 'rU') as fh:
             packages = [line.rstrip().split(':')
                         for line in fh]
         return packages