Bug 1304176 - Use vendored virtualenv if available; r=ted
authorGregory Szorc <gps@mozilla.com>
Tue, 20 Sep 2016 21:19:56 -0700
changeset 315182 3fd83c9c0548b3f07c844a2f6c3a23ae7569eecd
parent 315181 b64330d3db8d5091927308dc91ac1cfd5ec7b7ad
child 315183 8dc87cef62fb3dddae89f2a6a5099bb7dbaa6ebc
push id82104
push userihsiao@mozilla.com
push dateMon, 26 Sep 2016 11:08:56 +0000
treeherdermozilla-inbound@87aeaae19f90 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1304176
milestone52.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1304176 - Use vendored virtualenv if available; r=ted If mozharness is running from a source checkout, it has access to a modern virtualenv+pip/setuptools vendored as part of the source checkout. This commit changes the virtualenv creation code to use the vendored virtualenv when it is available. A side effect of this change is that a modern version of pip will now be used by mozharness when a source checkout is available. This has a number of consequences. First, modern versions of pip automatically create and cache wheels when building packages. This should make automation faster since it can now reuse cached wheels instead of having to download and rebuild packages all the time. Second, modern versions of pip support pinning package hashes. This opens the door to use having more secure package downloads and more determinism in our test environment. Third, modern versions of pip require connections to package servers be secure by default. Plaintext connections are disallowed by default. A --trusted-host option or environment variable can be used to override this behavior. Since upgrading pip resulted in some jobs failing due to disallowed connections to insecure servers, code to sniff the pip version and add --trusted-host where it is needed/supported. This retains the existing behavior. This is insecure. But fixing that is for another bug. As part of testing this, we were getting IOError inside virtualenv.py when installing distutils: IOError: [Errno 13] Permission denied: '/builds/slave/test/build/venv/lib/python2.7/distutils/__init__.py' We worked around this by adding --always-copy to the virtualenv.py invocation. MozReview-Commit-ID: D29ao9ZASei
testing/mozharness/mozharness/base/python.py
--- a/testing/mozharness/mozharness/base/python.py
+++ b/testing/mozharness/mozharness/base/python.py
@@ -2,22 +2,24 @@
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 '''Python usage, esp. virtualenv.
 '''
 
+import distutils.version
 import os
 import subprocess
 import sys
 import time
 import json
 import traceback
+import urlparse
 
 import mozharness
 from mozharness.base.script import (
     PostScriptAction,
     PostScriptRun,
     PreScriptAction,
     PreScriptRun,
 )
@@ -252,20 +254,30 @@ class VirtualenvMixin(object):
                 # the 'install' in the executable name hits UAC
                 # - http://answers.microsoft.com/en-us/windows/forum/windows_7-security/uac-message-do-you-want-to-allow-the-following/bea30ad8-9ef8-4897-aab4-841a65f7af71
                 # - https://bugzilla.mozilla.org/show_bug.cgi?id=791840
                 default = [self.query_python_path(), self.query_python_path('easy_install-script.py')]
             command = self.query_exe('easy_install', default=default, return_type="list")
         else:
             self.fatal("install_module() doesn't understand an install_method of %s!" % install_method)
 
-        # Add --find-links pages to look at
+        # Add --find-links pages to look at. Add --trusted-host automatically if
+        # the host isn't secure. This allows modern versions of pip to connect
+        # without requiring an override.
         proxxy = Proxxy(self.config, self.log_obj)
+        trusted_hosts = set()
         for link in proxxy.get_proxies_and_urls(c.get('find_links', [])):
             command.extend(["--find-links", link])
+            parsed = urlparse.urlparse(link)
+            if parsed.scheme != 'https':
+                trusted_hosts.add(parsed.hostname)
+
+        if self.pip_version >= distutils.version.LooseVersion('6.0'):
+            for host in sorted(trusted_hosts):
+                command.extend(['--trusted-host', host])
 
         # module_url can be None if only specifying requirements files
         if module_url:
             if editable:
                 if install_method in (None, 'pip'):
                     command += ['-e']
                 else:
                     self.fatal("editable installs not supported for install_method %s" % install_method)
@@ -344,55 +356,85 @@ class VirtualenvMixin(object):
                 '/path/to/requirements1.txt',
                 '/path/to/requirements2.txt'
             ]
         """
         c = self.config
         dirs = self.query_abs_dirs()
         venv_path = self.query_virtualenv_path()
         self.info("Creating virtualenv %s" % venv_path)
-        virtualenv = c.get('virtualenv', self.query_exe('virtualenv'))
-        if isinstance(virtualenv, str):
-            # allow for [python, virtualenv] in config
-            virtualenv = [virtualenv]
 
-        if not os.path.exists(virtualenv[0]) and not self.which(virtualenv[0]):
-            self.add_summary("The executable '%s' is not found; not creating "
-                             "virtualenv!" % virtualenv[0], level=FATAL)
-            return -1
+        # If running from a source checkout, use the virtualenv that is
+        # vendored since that is deterministic.
+        if self.topsrcdir:
+            virtualenv = [
+                sys.executable,
+                os.path.join(self.topsrcdir, 'python', 'virtualenv', 'virtualenv.py')
+            ]
+            virtualenv_options = c.get('virtualenv_options', [])
+            # Don't create symlinks. If we don't do this, permissions issues may
+            # hinder virtualenv creation or operation. Ideally we should do this
+            # below when using the system virtualenv. However, this is a newer
+            # feature and isn't guaranteed to be supported.
+            virtualenv_options.append('--always-copy')
+
+        # No source checkout. Try to find virtualenv from config options
+        # or search path.
+        else:
+            virtualenv = c.get('virtualenv', self.query_exe('virtualenv'))
+            if isinstance(virtualenv, str):
+                # allow for [python, virtualenv] in config
+                virtualenv = [virtualenv]
 
-        # https://bugs.launchpad.net/virtualenv/+bug/352844/comments/3
-        # https://bugzilla.mozilla.org/show_bug.cgi?id=700415#c50
-        if c.get('virtualenv_python_dll'):
-            # We may someday want to copy a differently-named dll, but
-            # let's not think about that right now =\
-            dll_name = os.path.basename(c['virtualenv_python_dll'])
-            target = self.query_python_path(dll_name)
-            scripts_dir = os.path.dirname(target)
-            self.mkdir_p(scripts_dir)
-            self.copyfile(c['virtualenv_python_dll'], target, error_level=WARNING)
-        else:
-            self.mkdir_p(dirs['abs_work_dir'])
+            if not os.path.exists(virtualenv[0]) and not self.which(virtualenv[0]):
+                self.add_summary("The executable '%s' is not found; not creating "
+                                 "virtualenv!" % virtualenv[0], level=FATAL)
+                return -1
 
-        # make this list configurable?
-        for module in ('distribute', 'pip'):
-            if c.get('%s_url' % module):
-                self.download_file(c['%s_url' % module],
-                                   parent_dir=dirs['abs_work_dir'])
+            # https://bugs.launchpad.net/virtualenv/+bug/352844/comments/3
+            # https://bugzilla.mozilla.org/show_bug.cgi?id=700415#c50
+            if c.get('virtualenv_python_dll'):
+                # We may someday want to copy a differently-named dll, but
+                # let's not think about that right now =\
+                dll_name = os.path.basename(c['virtualenv_python_dll'])
+                target = self.query_python_path(dll_name)
+                scripts_dir = os.path.dirname(target)
+                self.mkdir_p(scripts_dir)
+                self.copyfile(c['virtualenv_python_dll'], target, error_level=WARNING)
+            else:
+                self.mkdir_p(dirs['abs_work_dir'])
 
-        virtualenv_options = c.get('virtualenv_options',
-                                   ['--no-site-packages', '--distribute'])
+            # make this list configurable?
+            for module in ('distribute', 'pip'):
+                if c.get('%s_url' % module):
+                    self.download_file(c['%s_url' % module],
+                                       parent_dir=dirs['abs_work_dir'])
+
+            virtualenv_options = c.get('virtualenv_options',
+                                       ['--no-site-packages', '--distribute'])
 
         if os.path.exists(self.query_python_path()):
             self.info("Virtualenv %s appears to already exist; skipping virtualenv creation." % self.query_python_path())
         else:
             self.run_command(virtualenv + virtualenv_options + [venv_path],
                              cwd=dirs['abs_work_dir'],
                              error_list=VirtualenvErrorList,
                              halt_on_failure=True)
+
+        # Resolve the pip version so we can conditionally do things if we have
+        # a modern pip.
+        pip = self.query_python_path('pip')
+        output = self.get_output_from_command([pip, '--version'],
+                                              halt_on_failure=True)
+        words = output.split()
+        if words[0] != 'pip':
+            self.fatal('pip --version output is weird: %s' % output)
+        pip_version = words[1]
+        self.pip_version = distutils.version.LooseVersion(pip_version)
+
         if not modules:
             modules = c.get('virtualenv_modules', [])
         if not requirements:
             requirements = c.get('virtualenv_requirements', [])
         if not modules and requirements:
             self.install_module(requirements=requirements,
                                 install_method='pip')
         for module in modules: