Bug 1297515 - Always use the mozharness vendored copy of virtualenv. r=jlund, a=test-only
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 12 Oct 2017 12:24:05 -0400
changeset 432786 c5ed62b37c2e346ff2660eab13efa5c8657c6487
parent 432785 073a2a43a15cd103d2671b7e77d6700ddf3690f6
child 432787 d87eb6ee94579001e9729d91208c7c6df9dc4e06
push id8059
push userryanvm@gmail.com
push dateThu, 26 Oct 2017 16:45:03 +0000
treeherdermozilla-beta@c5ed62b37c2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjlund, test-only
bugs1297515
milestone57.0
Bug 1297515 - Always use the mozharness vendored copy of virtualenv. r=jlund, a=test-only This removes the option to pass in a custom virtualenv binary and gets mozharness to always use the copy in 'external_tools'. Because this is now deterministic, we don't need to do things like check what version of pip is being used. MozReview-Commit-ID: GWxkmaZVpsZ
testing/mozharness/mozharness/base/python.py
--- a/testing/mozharness/mozharness/base/python.py
+++ b/testing/mozharness/mozharness/base/python.py
@@ -2,17 +2,16 @@
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 '''Python usage, esp. virtualenv.
 '''
 
-import distutils.version
 import errno
 import os
 import subprocess
 import sys
 import json
 import socket
 import traceback
 import urlparse
@@ -53,21 +52,16 @@ def get_tlsv1_post():
 # Virtualenv {{{1
 virtualenv_config_options = [
     [["--virtualenv-path"], {
         "action": "store",
         "dest": "virtualenv_path",
         "default": "venv",
         "help": "Specify the path to the virtualenv top level directory"
     }],
-    [["--virtualenv"], {
-        "action": "store",
-        "dest": "virtualenv",
-        "help": "Specify the virtualenv executable to use"
-    }],
     [["--find-links"], {
         "action": "extend",
         "dest": "find_links",
         "help": "URL to look for packages at"
     }],
     [["--pip-index"], {
         "action": "store_true",
         "default": True,
@@ -246,22 +240,16 @@ class VirtualenvMixin(object):
             if requirements:
                 # Install pip requirements files separately, since they're
                 # not understood by easy_install.
                 self.install_module(requirements=requirements,
                                     install_method='pip')
             # Allow easy_install to be overridden by
             # self.config['exes']['easy_install']
             default = 'easy_install'
-            if self._is_windows():
-                # Don't invoke `easy_install` directly on windows since
-                # the 'install' in the executable name hits UAC
-                # - http://answers.microsoft.com/en-us/windows/forum/windows_7-security/uac-message-do-you-want-to-allow-the-following/bea30ad8-9ef8-4897-aab4-841a65f7af71
-                # - https://bugzilla.mozilla.org/show_bug.cgi?id=791840
-                default = [self.query_python_path(), self.query_python_path('easy_install-script.py')]
             command = self.query_exe('easy_install', default=default, return_type="list")
         else:
             self.fatal("install_module() doesn't understand an install_method of %s!" % install_method)
 
         # Add --find-links pages to look at. Add --trusted-host automatically if
         # the host isn't secure. This allows modern versions of pip to connect
         # without requiring an override.
         proxxy = Proxxy(self.config, self.log_obj)
@@ -275,18 +263,17 @@ class VirtualenvMixin(object):
                 self.info('error resolving %s (ignoring): %s' %
                           (parsed.hostname, e.message))
                 continue
 
             command.extend(["--find-links", link])
             if parsed.scheme != 'https':
                 trusted_hosts.add(parsed.hostname)
 
-        if (install_method != 'easy_install' and
-                    self.pip_version >= distutils.version.LooseVersion('6.0')):
+        if install_method != 'easy_install':
             for host in sorted(trusted_hosts):
                 command.extend(['--trusted-host', host])
 
         # module_url can be None if only specifying requirements files
         if module_url:
             if editable:
                 if install_method in (None, 'pip'):
                     command += ['-e']
@@ -368,84 +355,38 @@ class VirtualenvMixin(object):
                 '/path/to/requirements2.txt'
             ]
         """
         c = self.config
         dirs = self.query_abs_dirs()
         venv_path = self.query_virtualenv_path()
         self.info("Creating virtualenv %s" % venv_path)
 
-        # If running from a source checkout, use the virtualenv that is
-        # vendored since that is deterministic.
-        if self.topsrcdir:
-            virtualenv = [
-                sys.executable,
-                os.path.join(self.topsrcdir, 'third_party', 'python', 'virtualenv', 'virtualenv.py')
-            ]
-            virtualenv_options = c.get('virtualenv_options', [])
-            # Don't create symlinks. If we don't do this, permissions issues may
-            # hinder virtualenv creation or operation. Ideally we should do this
-            # below when using the system virtualenv. However, this is a newer
-            # feature and isn't guaranteed to be supported.
-            virtualenv_options.append('--always-copy')
-
-        # No source checkout. Try to find virtualenv from config options
-        # or search path.
-        else:
-            virtualenv = c.get('virtualenv', self.query_exe('virtualenv'))
-            if isinstance(virtualenv, str):
-                # allow for [python, virtualenv] in config
-                virtualenv = [virtualenv]
-
-            if not os.path.exists(virtualenv[0]) and not self.which(virtualenv[0]):
-                self.add_summary("The executable '%s' is not found; not creating "
-                                 "virtualenv!" % virtualenv[0], level=FATAL)
-                return -1
-
-            # https://bugs.launchpad.net/virtualenv/+bug/352844/comments/3
-            # https://bugzilla.mozilla.org/show_bug.cgi?id=700415#c50
-            if c.get('virtualenv_python_dll'):
-                # We may someday want to copy a differently-named dll, but
-                # let's not think about that right now =\
-                dll_name = os.path.basename(c['virtualenv_python_dll'])
-                target = self.query_python_path(dll_name)
-                scripts_dir = os.path.dirname(target)
-                self.mkdir_p(scripts_dir)
-                self.copyfile(c['virtualenv_python_dll'], target, error_level=WARNING)
-
-            # make this list configurable?
-            for module in ('distribute', 'pip'):
-                if c.get('%s_url' % module):
-                    self.download_file(c['%s_url' % module],
-                                       parent_dir=dirs['abs_work_dir'])
-
-            virtualenv_options = c.get('virtualenv_options',
-                                       ['--no-site-packages', '--distribute'])
+        # Always use the virtualenv that is vendored since that is deterministic.
+        # TODO Bug 1408051 - Use the copy of virtualenv under
+        # third_party/python/virtualenv once everything is off buildbot
+        virtualenv = [
+            sys.executable,
+            os.path.join(external_tools_path, 'virtualenv', 'virtualenv.py'),
+        ]
+        virtualenv_options = c.get('virtualenv_options', [])
+        # Don't create symlinks. If we don't do this, permissions issues may
+        # hinder virtualenv creation or operation.
+        virtualenv_options.append('--always-copy')
 
         if os.path.exists(self.query_python_path()):
             self.info("Virtualenv %s appears to already exist; skipping virtualenv creation." % self.query_python_path())
         else:
             self.mkdir_p(dirs['abs_work_dir'])
             self.run_command(virtualenv + virtualenv_options + [venv_path],
                              cwd=dirs['abs_work_dir'],
                              error_list=VirtualenvErrorList,
                              partial_env={'VIRTUALENV_NO_DOWNLOAD': "1"},
                              halt_on_failure=True)
 
-        # Resolve the pip version so we can conditionally do things if we have
-        # a modern pip.
-        pip = self.query_python_path('pip')
-        output = self.get_output_from_command([pip, '--version'],
-                                              halt_on_failure=True)
-        words = output.split()
-        if words[0] != 'pip':
-            self.fatal('pip --version output is weird: %s' % output)
-        pip_version = words[1]
-        self.pip_version = distutils.version.LooseVersion(pip_version)
-
         if not modules:
             modules = c.get('virtualenv_modules', [])
         if not requirements:
             requirements = c.get('virtualenv_requirements', [])
         if not modules and requirements:
             self.install_module(requirements=requirements,
                                 install_method='pip')
         for module in modules: