Bug 1255585 - Prevent Python executable mis-match from constantly clobbering the virtualenv on OS X. r=gps
authorChris Manchester <cmanchester@mozilla.com>
Fri, 11 Mar 2016 12:24:10 -0800
changeset 288618 a1a8876174c3258948d979707b2df1b913956902
parent 288617 ee3c99e6f6768fd804b58544efd97262cfea4779
child 288619 61b9a39b0bb49ce7fe957ca670ed2169bbc9ae1e
push id30087
push usercbook@mozilla.com
push dateTue, 15 Mar 2016 09:43:43 +0000
treeherdermozilla-central@5e14887312d4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1255585
milestone48.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1255585 - Prevent Python executable mis-match from constantly clobbering the virtualenv on OS X. r=gps Virtualenv will sometimes find a different executable from its sys.executable on OS X, causing a check in the build system comparing filesizes between sys.executable and virtualenv python to fail, resulting in clobbering and re-building the virtualenv every time the virtualenv is activated, causing the build backend and more to be re-built. Instead of checking file sizes directly, this commit causes us to record the size and version of the Python executable that created the virtualenv. If the Python executable checked is not the virtualenv Python, or we have a different version than was used to create the virtualenv, then the virtualenv is considered to be out of date. MozReview-Commit-ID: KmrVfQCtbS3
python/mozbuild/mozbuild/virtualenv.py
--- a/python/mozbuild/mozbuild/virtualenv.py
+++ b/python/mozbuild/mozbuild/virtualenv.py
@@ -45,16 +45,23 @@ class VirtualenvManager(object):
 
         Each manager is associated with a source directory, a path where you
         want the virtualenv to be created, and a handle to write output to.
         """
         assert os.path.isabs(manifest_path), "manifest_path must be an absolute path: %s" % (manifest_path)
         self.topsrcdir = topsrcdir
         self.topobjdir = topobjdir
         self.virtualenv_root = virtualenv_path
+
+        # Record the Python executable that was used to create the Virtualenv
+        # so we can check this against sys.executable when verifying the
+        # integrity of the virtualenv.
+        self.exe_info_path = os.path.join(self.virtualenv_root,
+                                          'python_exe.txt')
+
         self.log_handle = log_handle
         self.manifest_path = manifest_path
 
     @property
     def virtualenv_script_path(self):
         """Path to virtualenv's own populator script."""
         return os.path.join(self.topsrcdir, 'python', 'virtualenv',
             'virtualenv.py')
@@ -77,16 +84,35 @@ class VirtualenvManager(object):
             binary += '.exe'
 
         return os.path.join(self.bin_path, binary)
 
     @property
     def activate_path(self):
         return os.path.join(self.bin_path, 'activate_this.py')
 
+    def get_exe_info(self):
+        """Returns the version and file size of the python executable that was in
+        use when this virutalenv was created.
+        """
+        with open(self.exe_info_path, 'r') as fh:
+            version, size = fh.read().splitlines()
+        return int(version), int(size)
+
+    def write_exe_info(self, python):
+        """Records the the version of the python executable that was in use when
+        this virutalenv was created. We record this explicitly because
+        on OS X our python path may end up being a different or modified
+        executable.
+        """
+        ver = subprocess.check_output([python, '-c', 'import sys; print(sys.hexversion)']).rstrip()
+        with open(self.exe_info_path, 'w') as fh:
+            fh.write("%s\n" % ver)
+            fh.write("%s\n" % os.path.getsize(python))
+
     def up_to_date(self, python=sys.executable):
         """Returns whether the virtualenv is present and up to date."""
 
         deps = [self.manifest_path, __file__]
 
         # check if virtualenv exists
         if not os.path.exists(self.virtualenv_root) or \
             not os.path.exists(self.activate_path):
@@ -94,20 +120,23 @@ class VirtualenvManager(object):
             return False
 
         # check modification times
         activate_mtime = os.path.getmtime(self.activate_path)
         dep_mtime = max(os.path.getmtime(p) for p in deps)
         if dep_mtime > activate_mtime:
             return False
 
-        # check interpreter. Assume that a different size is enough to
-        # know whether we've been given a different interpreter, and thus
-        # should refresh the virtualenv.
-        if os.path.getsize(python) != os.path.getsize(self.python_path):
+        # Verify that the Python we're checking here is either the virutalenv
+        # python, or we have the Python version that was used to create the
+        # virtualenv. If this fails, it is likely system Python has been
+        # upgraded, and our virtualenv would not be usable.
+        python_size = os.path.getsize(python)
+        if ((python, python_size) != (self.python_path, os.path.getsize(self.python_path)) and
+            (sys.hexversion, python_size) != self.get_exe_info()):
             return False
 
         # recursively check sub packages.txt files
         submanifests = [i[1] for i in self.packages()
                         if i[0] == 'packages.txt']
         for submanifest in submanifests:
             submanifest = os.path.join(self.topsrcdir, submanifest)
             submanager = VirtualenvManager(self.topsrcdir,
@@ -148,16 +177,18 @@ class VirtualenvManager(object):
 
         result = subprocess.call(args, stdout=self.log_handle,
             stderr=subprocess.STDOUT, env=env)
 
         if result:
             raise Exception(
                 'Failed to create virtualenv: %s' % self.virtualenv_root)
 
+        self.write_exe_info(python)
+
         return self.virtualenv_root
 
     def packages(self):
         with file(self.manifest_path, 'rU') as fh:
             packages = [line.rstrip().split(':')
                         for line in fh]
         return packages