author | Gregory Szorc <gps@mozilla.com> |
Tue, 20 Sep 2016 21:19:56 -0700 | |
changeset 315182 | 3fd83c9c0548b3f07c844a2f6c3a23ae7569eecd |
parent 315181 | b64330d3db8d5091927308dc91ac1cfd5ec7b7ad |
child 315183 | 8dc87cef62fb3dddae89f2a6a5099bb7dbaa6ebc |
push id | 82104 |
push user | ihsiao@mozilla.com |
push date | Mon, 26 Sep 2016 11:08:56 +0000 |
treeherder | mozilla-inbound@87aeaae19f90 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | ted |
bugs | 1304176 |
milestone | 52.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/testing/mozharness/mozharness/base/python.py +++ b/testing/mozharness/mozharness/base/python.py @@ -2,22 +2,24 @@ # ***** BEGIN LICENSE BLOCK ***** # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. # ***** END LICENSE BLOCK ***** '''Python usage, esp. virtualenv. ''' +import distutils.version import os import subprocess import sys import time import json import traceback +import urlparse import mozharness from mozharness.base.script import ( PostScriptAction, PostScriptRun, PreScriptAction, PreScriptRun, ) @@ -252,20 +254,30 @@ class VirtualenvMixin(object): # the 'install' in the executable name hits UAC # - http://answers.microsoft.com/en-us/windows/forum/windows_7-security/uac-message-do-you-want-to-allow-the-following/bea30ad8-9ef8-4897-aab4-841a65f7af71 # - https://bugzilla.mozilla.org/show_bug.cgi?id=791840 default = [self.query_python_path(), self.query_python_path('easy_install-script.py')] command = self.query_exe('easy_install', default=default, return_type="list") else: self.fatal("install_module() doesn't understand an install_method of %s!" % install_method) - # Add --find-links pages to look at + # Add --find-links pages to look at. Add --trusted-host automatically if + # the host isn't secure. This allows modern versions of pip to connect + # without requiring an override. proxxy = Proxxy(self.config, self.log_obj) + trusted_hosts = set() for link in proxxy.get_proxies_and_urls(c.get('find_links', [])): command.extend(["--find-links", link]) + parsed = urlparse.urlparse(link) + if parsed.scheme != 'https': + trusted_hosts.add(parsed.hostname) + + if self.pip_version >= distutils.version.LooseVersion('6.0'): + for host in sorted(trusted_hosts): + command.extend(['--trusted-host', host]) # module_url can be None if only specifying requirements files if module_url: if editable: if install_method in (None, 'pip'): command += ['-e'] else: self.fatal("editable installs not supported for install_method %s" % install_method) @@ -344,55 +356,85 @@ class VirtualenvMixin(object): '/path/to/requirements1.txt', '/path/to/requirements2.txt' ] """ c = self.config dirs = self.query_abs_dirs() venv_path = self.query_virtualenv_path() self.info("Creating virtualenv %s" % venv_path) - virtualenv = c.get('virtualenv', self.query_exe('virtualenv')) - if isinstance(virtualenv, str): - # allow for [python, virtualenv] in config - virtualenv = [virtualenv] - if not os.path.exists(virtualenv[0]) and not self.which(virtualenv[0]): - self.add_summary("The executable '%s' is not found; not creating " - "virtualenv!" % virtualenv[0], level=FATAL) - return -1 + # If running from a source checkout, use the virtualenv that is + # vendored since that is deterministic. + if self.topsrcdir: + virtualenv = [ + sys.executable, + os.path.join(self.topsrcdir, 'python', 'virtualenv', 'virtualenv.py') + ] + virtualenv_options = c.get('virtualenv_options', []) + # Don't create symlinks. If we don't do this, permissions issues may + # hinder virtualenv creation or operation. Ideally we should do this + # below when using the system virtualenv. However, this is a newer + # feature and isn't guaranteed to be supported. + virtualenv_options.append('--always-copy') + + # No source checkout. Try to find virtualenv from config options + # or search path. + else: + virtualenv = c.get('virtualenv', self.query_exe('virtualenv')) + if isinstance(virtualenv, str): + # allow for [python, virtualenv] in config + virtualenv = [virtualenv] - # https://bugs.launchpad.net/virtualenv/+bug/352844/comments/3 - # https://bugzilla.mozilla.org/show_bug.cgi?id=700415#c50 - if c.get('virtualenv_python_dll'): - # We may someday want to copy a differently-named dll, but - # let's not think about that right now =\ - dll_name = os.path.basename(c['virtualenv_python_dll']) - target = self.query_python_path(dll_name) - scripts_dir = os.path.dirname(target) - self.mkdir_p(scripts_dir) - self.copyfile(c['virtualenv_python_dll'], target, error_level=WARNING) - else: - self.mkdir_p(dirs['abs_work_dir']) + if not os.path.exists(virtualenv[0]) and not self.which(virtualenv[0]): + self.add_summary("The executable '%s' is not found; not creating " + "virtualenv!" % virtualenv[0], level=FATAL) + return -1 - # make this list configurable? - for module in ('distribute', 'pip'): - if c.get('%s_url' % module): - self.download_file(c['%s_url' % module], - parent_dir=dirs['abs_work_dir']) + # https://bugs.launchpad.net/virtualenv/+bug/352844/comments/3 + # https://bugzilla.mozilla.org/show_bug.cgi?id=700415#c50 + if c.get('virtualenv_python_dll'): + # We may someday want to copy a differently-named dll, but + # let's not think about that right now =\ + dll_name = os.path.basename(c['virtualenv_python_dll']) + target = self.query_python_path(dll_name) + scripts_dir = os.path.dirname(target) + self.mkdir_p(scripts_dir) + self.copyfile(c['virtualenv_python_dll'], target, error_level=WARNING) + else: + self.mkdir_p(dirs['abs_work_dir']) - virtualenv_options = c.get('virtualenv_options', - ['--no-site-packages', '--distribute']) + # make this list configurable? + for module in ('distribute', 'pip'): + if c.get('%s_url' % module): + self.download_file(c['%s_url' % module], + parent_dir=dirs['abs_work_dir']) + + virtualenv_options = c.get('virtualenv_options', + ['--no-site-packages', '--distribute']) if os.path.exists(self.query_python_path()): self.info("Virtualenv %s appears to already exist; skipping virtualenv creation." % self.query_python_path()) else: self.run_command(virtualenv + virtualenv_options + [venv_path], cwd=dirs['abs_work_dir'], error_list=VirtualenvErrorList, halt_on_failure=True) + + # Resolve the pip version so we can conditionally do things if we have + # a modern pip. + pip = self.query_python_path('pip') + output = self.get_output_from_command([pip, '--version'], + halt_on_failure=True) + words = output.split() + if words[0] != 'pip': + self.fatal('pip --version output is weird: %s' % output) + pip_version = words[1] + self.pip_version = distutils.version.LooseVersion(pip_version) + if not modules: modules = c.get('virtualenv_modules', []) if not requirements: requirements = c.get('virtualenv_requirements', []) if not modules and requirements: self.install_module(requirements=requirements, install_method='pip') for module in modules: