Bug 1297515 - Always use the mozharness vendored copy of virtualenv, r=jlund
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 12 Oct 2017 12:24:05 -0400
changeset 386806 5a44c9c16c21e3d1a285a97be2447c9c9e2f1f98
parent 386805 da7ac021e1209f76295cd19594280002d5e6ef38
child 386807 3e050546fe966ee1afce6e24e3dadfaa146133a3
push id96311
push userarchaeopteryx@coole-files.de
push dateWed, 18 Oct 2017 09:52:02 +0000
treeherdermozilla-inbound@a8a1e8cc1980 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjlund
bugs1297515
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1297515 - Always use the mozharness vendored copy of virtualenv, r=jlund This removes the option to pass in a custom virtualenv binary and gets mozharness to always use the copy in 'external_tools'. Because this is now deterministic, we don't need to do things like check what version of pip is being used. MozReview-Commit-ID: GWxkmaZVpsZ
testing/mozharness/mozharness/base/python.py
--- a/testing/mozharness/mozharness/base/python.py
+++ b/testing/mozharness/mozharness/base/python.py
@@ -2,17 +2,16 @@
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 '''Python usage, esp. virtualenv.
 '''
 
-import distutils.version
 import errno
 import os
 import subprocess
 import sys
 import json
 import socket
 import traceback
 import urlparse
@@ -53,21 +52,16 @@ def get_tlsv1_post():
 # Virtualenv {{{1
 virtualenv_config_options = [
     [["--virtualenv-path"], {
         "action": "store",
         "dest": "virtualenv_path",
         "default": "venv",
         "help": "Specify the path to the virtualenv top level directory"
     }],
-    [["--virtualenv"], {
-        "action": "store",
-        "dest": "virtualenv",
-        "help": "Specify the virtualenv executable to use"
-    }],
     [["--find-links"], {
         "action": "extend",
         "dest": "find_links",
         "help": "URL to look for packages at"
     }],
     [["--pip-index"], {
         "action": "store_true",
         "default": True,
@@ -246,22 +240,16 @@ class VirtualenvMixin(object):
             if requirements:
                 # Install pip requirements files separately, since they're
                 # not understood by easy_install.
                 self.install_module(requirements=requirements,
                                     install_method='pip')
             # Allow easy_install to be overridden by
             # self.config['exes']['easy_install']
             default = 'easy_install'
-            if self._is_windows():
-                # Don't invoke `easy_install` directly on windows since
-                # the 'install' in the executable name hits UAC
-                # - http://answers.microsoft.com/en-us/windows/forum/windows_7-security/uac-message-do-you-want-to-allow-the-following/bea30ad8-9ef8-4897-aab4-841a65f7af71
-                # - https://bugzilla.mozilla.org/show_bug.cgi?id=791840
-                default = [self.query_python_path(), self.query_python_path('easy_install-script.py')]
             command = self.query_exe('easy_install', default=default, return_type="list")
         else:
             self.fatal("install_module() doesn't understand an install_method of %s!" % install_method)
 
         # Add --find-links pages to look at. Add --trusted-host automatically if
         # the host isn't secure. This allows modern versions of pip to connect
         # without requiring an override.
         proxxy = Proxxy(self.config, self.log_obj)
@@ -275,18 +263,17 @@ class VirtualenvMixin(object):
                 self.info('error resolving %s (ignoring): %s' %
                           (parsed.hostname, e.message))
                 continue
 
             command.extend(["--find-links", link])
             if parsed.scheme != 'https':
                 trusted_hosts.add(parsed.hostname)
 
-        if (install_method != 'easy_install' and
-                    self.pip_version >= distutils.version.LooseVersion('6.0')):
+        if install_method != 'easy_install':
             for host in sorted(trusted_hosts):
                 command.extend(['--trusted-host', host])
 
         # module_url can be None if only specifying requirements files
         if module_url:
             if editable:
                 if install_method in (None, 'pip'):
                     command += ['-e']
@@ -368,84 +355,38 @@ class VirtualenvMixin(object):
                 '/path/to/requirements2.txt'
             ]
         """
         c = self.config
         dirs = self.query_abs_dirs()
         venv_path = self.query_virtualenv_path()
         self.info("Creating virtualenv %s" % venv_path)
 
-        # If running from a source checkout, use the virtualenv that is
-        # vendored since that is deterministic.
-        if self.topsrcdir:
-            virtualenv = [
-                sys.executable,
-                os.path.join(self.topsrcdir, 'third_party', 'python', 'virtualenv', 'virtualenv.py')
-            ]
-            virtualenv_options = c.get('virtualenv_options', [])
-            # Don't create symlinks. If we don't do this, permissions issues may
-            # hinder virtualenv creation or operation. Ideally we should do this
-            # below when using the system virtualenv. However, this is a newer
-            # feature and isn't guaranteed to be supported.
-            virtualenv_options.append('--always-copy')
-
-        # No source checkout. Try to find virtualenv from config options
-        # or search path.
-        else:
-            virtualenv = c.get('virtualenv', self.query_exe('virtualenv'))
-            if isinstance(virtualenv, str):
-                # allow for [python, virtualenv] in config
-                virtualenv = [virtualenv]
-
-            if not os.path.exists(virtualenv[0]) and not self.which(virtualenv[0]):
-                self.add_summary("The executable '%s' is not found; not creating "
-                                 "virtualenv!" % virtualenv[0], level=FATAL)
-                return -1
-
-            # https://bugs.launchpad.net/virtualenv/+bug/352844/comments/3
-            # https://bugzilla.mozilla.org/show_bug.cgi?id=700415#c50
-            if c.get('virtualenv_python_dll'):
-                # We may someday want to copy a differently-named dll, but
-                # let's not think about that right now =\
-                dll_name = os.path.basename(c['virtualenv_python_dll'])
-                target = self.query_python_path(dll_name)
-                scripts_dir = os.path.dirname(target)
-                self.mkdir_p(scripts_dir)
-                self.copyfile(c['virtualenv_python_dll'], target, error_level=WARNING)
-
-            # make this list configurable?
-            for module in ('distribute', 'pip'):
-                if c.get('%s_url' % module):
-                    self.download_file(c['%s_url' % module],
-                                       parent_dir=dirs['abs_work_dir'])
-
-            virtualenv_options = c.get('virtualenv_options',
-                                       ['--no-site-packages', '--distribute'])
+        # Always use the virtualenv that is vendored since that is deterministic.
+        # TODO Bug 1408051 - Use the copy of virtualenv under
+        # third_party/python/virtualenv once everything is off buildbot
+        virtualenv = [
+            sys.executable,
+            os.path.join(external_tools_path, 'virtualenv', 'virtualenv.py'),
+        ]
+        virtualenv_options = c.get('virtualenv_options', [])
+        # Don't create symlinks. If we don't do this, permissions issues may
+        # hinder virtualenv creation or operation.
+        virtualenv_options.append('--always-copy')
 
         if os.path.exists(self.query_python_path()):
             self.info("Virtualenv %s appears to already exist; skipping virtualenv creation." % self.query_python_path())
         else:
             self.mkdir_p(dirs['abs_work_dir'])
             self.run_command(virtualenv + virtualenv_options + [venv_path],
                              cwd=dirs['abs_work_dir'],
                              error_list=VirtualenvErrorList,
                              partial_env={'VIRTUALENV_NO_DOWNLOAD': "1"},
                              halt_on_failure=True)
 
-        # Resolve the pip version so we can conditionally do things if we have
-        # a modern pip.
-        pip = self.query_python_path('pip')
-        output = self.get_output_from_command([pip, '--version'],
-                                              halt_on_failure=True)
-        words = output.split()
-        if words[0] != 'pip':
-            self.fatal('pip --version output is weird: %s' % output)
-        pip_version = words[1]
-        self.pip_version = distutils.version.LooseVersion(pip_version)
-
         if not modules:
             modules = c.get('virtualenv_modules', [])
         if not requirements:
             requirements = c.get('virtualenv_requirements', [])
         if not modules and requirements:
             self.install_module(requirements=requirements,
                                 install_method='pip')
         for module in modules: