Bug 1466211 - Switch all |mach python-test| tests to run using pipenv; r=ahal
authorDave Hunt <dhunt@mozilla.com>
Fri, 08 Jun 2018 13:24:27 +0100
changeset 423533 8118894d2053c571f07aeb7f95e0b9717025fa53
parent 423532 f761d8aed904b846ec3daa453d7ede79d2357a19
child 423534 9b0a35f0572a9c809f1e6d12a40c5532cdf7428e
push id65490
push userdhunt@mozilla.com
push dateMon, 25 Jun 2018 15:29:04 +0000
treeherderautoland@434d589d020d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1466211
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1466211 - Switch all |mach python-test| tests to run using pipenv; r=ahal MozReview-Commit-ID: AzmdDgAgZgI
Pipfile
Pipfile.lock
python/Pipfile
python/Pipfile.lock
python/mach_commands.py
python/mozbuild/mozbuild/base.py
python/mozbuild/mozbuild/pipenv.txt
python/mozbuild/mozbuild/virtualenv.py
--- a/Pipfile
+++ b/Pipfile
@@ -11,12 +11,8 @@ jsmin = "==2.1.0"
 json-e = "==2.5.0"
 pipenv = "==2018.5.18"
 pytest = "==3.2.5"
 python-hglib = "==2.4"
 requests = "==2.9.1"
 six = "==1.10.0"
 virtualenv = "==15.2.0"
 voluptuous = "==0.10.5"
-
-
-[requires]
-python_version = "2.7"
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,17 +1,15 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "af4e239c88ce3d74e2e3dd7d352c3e8a203ce476c7369b2a4dc0eea7114996ba"
+            "sha256": "eb8b0a9771d4420f83fbbabf9952dc783aeefe9be455559de2f3ebff27caa93f"
         },
         "pipfile-spec": 6,
-        "requires": {
-            "python_version": "2.7"
-        },
+        "requires": {},
         "sources": [
             {
                 "name": "pypi",
                 "url": "https://pypi.org/simple",
                 "verify_ssl": true
             }
         ]
     },
deleted file mode 100644
--- a/python/Pipfile
+++ /dev/null
@@ -1,36 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-"d5b4a14" = {path = "./mach"}
-"8ddb376" = {path = "./mozbuild"}
-"b3ddbcf" = {path = "./mozterm"}
-"38a4a9a" = {path = "./mozversioncontrol"}
-"26d92fb" = {path = "./../config/mozunit"}
-"cea2946" = {path = "./../testing/mozbase/manifestparser"}
-"ffcf6e6" = {path = "./../testing/mozbase/mozcrash"}
-"195ae2e" = {path = "./../testing/mozbase/mozdebug"}
-"8dab59a" = {path = "./../testing/mozbase/mozdevice"}
-"58d0848" = {path = "./../testing/mozbase/mozfile"}
-"fd0b608" = {path = "./../testing/mozbase/mozhttpd"}
-"7329809" = {path = "./../testing/mozbase/mozinfo"}
-"501835d" = {path = "./../testing/mozbase/mozinstall"}
-"807c1c5" = {path = "./../testing/mozbase/mozlog"}
-"e09e103" = {path = "./../testing/mozbase/moznetwork"}
-"132adec" = {path = "./../testing/mozbase/mozprocess"}
-"d88f467" = {path = "./../testing/mozbase/mozprofile"}
-"1de94f2" = {path = "./../testing/mozbase/mozrunner"}
-"6477f20" = {path = "./../testing/mozbase/moztest"}
-"f1d74ca" = {path = "./../testing/mozbase/mozversion"}
-"47200d8" = {path = "./../third_party/python/futures", markers="python_version < '3'"}
-"110bcc4" = {path = "./../third_party/python/jsmin"}
-"c49d32a" = {path = "./../third_party/python/mock-1.0.0", markers="python_version < '3.3'"}
-"c2c21d9" = {path = "./../third_party/python/py"}
-"f4b00e9" = {path = "./../third_party/python/pytest"}
-"053111f" = {path = "./../third_party/python/requests"}
-"d250320" = {path = "./../third_party/python/six"}
-"f1de77a" = {path = "./../third_party/python/which", markers="python_version < '3.3'"}
-
-[dev-packages]
deleted file mode 100644
--- a/python/Pipfile.lock
+++ /dev/null
@@ -1,106 +0,0 @@
-{
-    "_meta": {
-        "hash": {
-            "sha256": "dfc219f64edc7715acdb35e03dcee665ec26908c18a58d3a3a88dda3ab393b17"
-        },
-        "pipfile-spec": 6,
-        "requires": {},
-        "sources": [
-            {
-                "name": "pypi",
-                "url": "https://pypi.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {
-        "053111f": {
-            "path": "./../third_party/python/requests"
-        },
-        "110bcc4": {
-            "path": "./../third_party/python/jsmin"
-        },
-        "132adec": {
-            "path": "./../testing/mozbase/mozprocess"
-        },
-        "195ae2e": {
-            "path": "./../testing/mozbase/mozdebug"
-        },
-        "1de94f2": {
-            "path": "./../testing/mozbase/mozrunner"
-        },
-        "26d92fb": {
-            "path": "./../config/mozunit"
-        },
-        "38a4a9a": {
-            "path": "./mozversioncontrol"
-        },
-        "47200d8": {
-            "markers": "python_version < '3'",
-            "path": "./../third_party/python/futures"
-        },
-        "501835d": {
-            "path": "./../testing/mozbase/mozinstall"
-        },
-        "58d0848": {
-            "path": "./../testing/mozbase/mozfile"
-        },
-        "6477f20": {
-            "path": "./../testing/mozbase/moztest"
-        },
-        "7329809": {
-            "path": "./../testing/mozbase/mozinfo"
-        },
-        "807c1c5": {
-            "path": "./../testing/mozbase/mozlog"
-        },
-        "8dab59a": {
-            "path": "./../testing/mozbase/mozdevice"
-        },
-        "8ddb376": {
-            "path": "./mozbuild"
-        },
-        "b3ddbcf": {
-            "path": "./mozterm"
-        },
-        "c2c21d9": {
-            "path": "./../third_party/python/py"
-        },
-        "c49d32a": {
-            "markers": "python_version < '3.3'",
-            "path": "./../third_party/python/mock-1.0.0"
-        },
-        "cea2946": {
-            "path": "./../testing/mozbase/manifestparser"
-        },
-        "d250320": {
-            "path": "./../third_party/python/six"
-        },
-        "d5b4a14": {
-            "path": "./mach"
-        },
-        "d88f467": {
-            "path": "./../testing/mozbase/mozprofile"
-        },
-        "e09e103": {
-            "path": "./../testing/mozbase/moznetwork"
-        },
-        "f1d74ca": {
-            "path": "./../testing/mozbase/mozversion"
-        },
-        "f1de77a": {
-            "markers": "python_version < '3.3'",
-            "path": "./../third_party/python/which"
-        },
-        "f4b00e9": {
-            "path": "./../third_party/python/pytest"
-        },
-        "fd0b608": {
-            "path": "./../testing/mozbase/mozhttpd"
-        },
-        "ffcf6e6": {
-            "path": "./../testing/mozbase/mozcrash"
-        }
-    },
-    "develop": {}
-}
--- a/python/mach_commands.py
+++ b/python/mach_commands.py
@@ -94,21 +94,18 @@ class MachCommands(MachCommandBase):
     def run_python_tests(self,
                          tests=None,
                          test_objects=None,
                          subsuite=None,
                          verbose=False,
                          jobs=1,
                          three=False,
                          **kwargs):
-        if three:
-            # use pipenv to run tests against Python 3
-            self.activate_pipenv(os.path.join(here, 'Pipfile'), ['--three'])
-        else:
-            self._activate_virtualenv()
+        pipenv_args = ['--three' if three else '--two']
+        self.activate_pipenv(pipfile=None, args=pipenv_args, populate=True)
 
         if test_objects is None:
             from moztest.resolve import TestResolver
             resolver = self._spawn(TestResolver)
             # If we were given test paths, try to find tests matching them.
             test_objects = resolver.resolve_tests(paths=tests, flavor='python')
         else:
             # We've received test_objects from |mach test|. We need to ignore
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -747,26 +747,26 @@ class MozbuildObject(ProcessExecutionMix
 
     def _set_log_level(self, verbose):
         self.log_manager.terminal_handler.setLevel(logging.INFO if not verbose else logging.DEBUG)
 
     def ensure_pipenv(self):
         self._activate_virtualenv()
         pipenv = os.path.join(self.virtualenv_manager.bin_path, 'pipenv')
         if not os.path.exists(pipenv):
-            pipenv_reqs = os.path.join(self.topsrcdir, 'python/mozbuild/mozbuild/pipenv.txt')
-            self.virtualenv_manager.install_pip_requirements(
-                pipenv_reqs, require_hashes=False, vendored=True)
+            for package in ['certifi', 'pipenv', 'six', 'virtualenv', 'virtualenv-clone']:
+                path = os.path.normpath(os.path.join(self.topsrcdir, 'third_party/python', package))
+                self.virtualenv_manager.install_pip_package(path, vendored=True)
         return pipenv
 
-    def activate_pipenv(self, path, args=None):
-        if not os.path.exists(path):
-            raise Exception('Pipfile not found: %s.' % path)
+    def activate_pipenv(self, pipfile=None, args=None, populate=False):
+        if pipfile is not None and not os.path.exists(pipfile):
+            raise Exception('Pipfile not found: %s.' % pipfile)
         self.ensure_pipenv()
-        self.virtualenv_manager.activate_pipenv(path, args)
+        self.virtualenv_manager.activate_pipenv(pipfile, args, populate)
 
 
 class MachCommandBase(MozbuildObject):
     """Base class for mach command providers that wish to be MozbuildObjects.
 
     This provides a level of indirection so MozbuildObject can be refactored
     without having to change everything that inherits from it.
     """
deleted file mode 100644
--- a/python/mozbuild/mozbuild/pipenv.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-third_party/python/certifi
-third_party/python/pipenv
-third_party/python/six
-third_party/python/virtualenv
-third_party/python/virtualenv-clone
--- a/python/mozbuild/mozbuild/virtualenv.py
+++ b/python/mozbuild/mozbuild/virtualenv.py
@@ -15,34 +15,38 @@ import sys
 import warnings
 
 from distutils.version import LooseVersion
 
 IS_NATIVE_WIN = (sys.platform == 'win32' and os.sep == '\\')
 IS_MSYS2 = (sys.platform == 'win32' and os.sep == '/')
 IS_CYGWIN = (sys.platform == 'cygwin')
 
-# Minimum version of Python required to build.
-MINIMUM_PYTHON_VERSION = LooseVersion('2.7.3')
-MINIMUM_PYTHON_MAJOR = 2
+# Minimum versions of Python required to build.
+MINIMUM_PYTHON_VERSIONS = {
+    2: LooseVersion('2.7.3'),
+    3: LooseVersion('3.5.0')
+}
 
 
 UPGRADE_WINDOWS = '''
 Please upgrade to the latest MozillaBuild development environment. See
 https://developer.mozilla.org/en-US/docs/Developer_Guide/Build_Instructions/Windows_Prerequisites
 '''.lstrip()
 
 UPGRADE_OTHER = '''
 Run |mach bootstrap| to ensure your system is up to date.
 
 If you still receive this error, your shell environment is likely detecting
 another Python version. Ensure a modern Python can be found in the paths
 defined by the $PATH environment variable and try again.
 '''.lstrip()
 
+here = os.path.abspath(os.path.dirname(__file__))
+
 
 class VirtualenvManager(object):
     """Contains logic for managing virtualenvs for building the tree."""
 
     def __init__(self, topsrcdir, topobjdir, virtualenv_path, log_handle,
         manifest_path):
         """Create a new manager.
 
@@ -202,17 +206,17 @@ class VirtualenvManager(object):
             raise Exception(
                 'Failed to create virtualenv: %s' % self.virtualenv_root)
 
         self.write_exe_info(python)
 
         return self.virtualenv_root
 
     def packages(self):
-        with file(self.manifest_path, 'rU') as fh:
+        with open(self.manifest_path, 'rU') as fh:
             packages = [line.rstrip().split(':')
                         for line in fh]
         return packages
 
     def populate(self):
         """Populate the virtualenv.
 
         The manifest file consists of colon-delimited fields. The first field
@@ -245,17 +249,16 @@ class VirtualenvManager(object):
             search path. e.g. "objdir:build" will add $topobjdir/build to the
             search path.
 
         Note that the Python interpreter running this function should be the
         one from the virtualenv. If it is the system Python or if the
         environment is not configured properly, packages could be installed
         into the wrong place. This is how virtualenv's work.
         """
-
         packages = self.packages()
         python_lib = distutils.sysconfig.get_python_lib()
 
         def handle_package(package):
             if package[0] == 'setup.py':
                 assert len(package) >= 2
 
                 self.call_setup(os.path.join(self.topsrcdir, package[1]),
@@ -460,37 +463,46 @@ class VirtualenvManager(object):
         virtualenv, you can simply instantiate an instance of this class
         and call .ensure() and .activate() to make the virtualenv active.
         """
 
         execfile(self.activate_path, dict(__file__=self.activate_path))
         if isinstance(os.environ['PATH'], unicode):
             os.environ['PATH'] = os.environ['PATH'].encode('utf-8')
 
-    def install_pip_package(self, package):
+    def install_pip_package(self, package, vendored=False):
         """Install a package via pip.
 
         The supplied package is specified using a pip requirement specifier.
         e.g. 'foo' or 'foo==1.0'.
 
         If the package is already installed, this is a no-op.
+
+        If vendored is True, no package index will be used and no dependencies
+        will be installed.
         """
         from pip.req import InstallRequirement
 
         req = InstallRequirement.from_line(package)
         req.check_if_exists()
         if req.satisfied_by is not None:
             return
 
         args = [
             'install',
             '--use-wheel',
             package,
         ]
 
+        if vendored:
+            args.extend([
+                '--no-deps',
+                '--no-index',
+            ])
+
         return self._run_pip(args)
 
     def install_pip_requirements(self, path, require_hashes=True, quiet=False, vendored=False):
         """Install a pip requirements.txt file.
 
         The supplied path is a text file containing pip requirement
         specifiers.
 
@@ -525,52 +537,72 @@ class VirtualenvManager(object):
     def _run_pip(self, args):
         # It's tempting to call pip natively via pip.main(). However,
         # the current Python interpreter may not be the virtualenv python.
         # This will confuse pip and cause the package to attempt to install
         # against the executing interpreter. By creating a new process, we
         # force the virtualenv's interpreter to be used and all is well.
         # It /might/ be possible to cheat and set sys.executable to
         # self.python_path. However, this seems more risk than it's worth.
-        subprocess.check_call([os.path.join(self.bin_path, 'pip')] + args,
-            stderr=subprocess.STDOUT)
+        pip = os.path.join(self.bin_path, 'pip')
+        subprocess.check_call([pip] + args, stderr=subprocess.STDOUT, cwd=self.topsrcdir)
+
+    def activate_pipenv(self, pipfile=None, args=None, populate=False):
+        """Activate a virtual environment managed by pipenv
 
-    def activate_pipenv(self, pipfile, args=None):
-        """Install a Pipfile located at path and activate environment"""
+        If ``pipfile`` is not ``None`` then the Pipfile located at the path
+        provided will be used to create the virtual environment. If
+        ``populate`` is ``True`` then the virtual environment will be
+        populated from the manifest file. The optional ``args`` list will be
+        passed to the pipenv commands.
+        """
         pipenv = os.path.join(self.bin_path, 'pipenv')
         env = os.environ.copy()
         env.update({
-            'PIPENV_IGNORE_VIRTUALENVS': '1',
-            'PIPENV_PIPFILE': pipfile,
-            'WORKON_HOME': os.path.join(self.topobjdir, '_virtualenvs'),
+            b'PIPENV_IGNORE_VIRTUALENVS': b'1',
+            b'WORKON_HOME': str(os.path.normpath(os.path.join(self.topobjdir, '_virtualenvs'))),
         })
 
         args = args or []
+
+        if pipfile is not None:
+            # Install from Pipfile
+            env[b'PIPENV_PIPFILE'] = str(pipfile)
+            args.append('install')
+
         subprocess.check_call(
-            [pipenv, 'install'] + args,
+            [pipenv] + args,
             stderr=subprocess.STDOUT,
             env=env)
 
         self.virtualenv_root = subprocess.check_output(
             [pipenv, '--venv'],
             stderr=subprocess.STDOUT,
             env=env).rstrip()
 
+        if populate:
+            # Populate from the manifest
+            subprocess.check_call([
+                pipenv, 'run', 'python', os.path.join(here, 'virtualenv.py'), 'populate',
+                self.topsrcdir, self.topobjdir, self.virtualenv_root, self.manifest_path],
+                stderr=subprocess.STDOUT, env=env)
+
         self.activate()
 
 
 def verify_python_version(log_handle):
     """Ensure the current version of Python is sufficient."""
     major, minor, micro = sys.version_info[:3]
 
     our = LooseVersion('%d.%d.%d' % (major, minor, micro))
 
-    if major != MINIMUM_PYTHON_MAJOR or our < MINIMUM_PYTHON_VERSION:
-        log_handle.write('Python %s or greater (but not Python 3) is '
-            'required to build. ' % MINIMUM_PYTHON_VERSION)
+    if major not in MINIMUM_PYTHON_VERSIONS or our < MINIMUM_PYTHON_VERSIONS[major]:
+        log_handle.write('One of the following Python versions are required to build:\n')
+        for minver in MINIMUM_PYTHON_VERSIONS.values():
+            log_handle.write('* Python %s or greater\n' % minver)
         log_handle.write('You are running Python %s.\n' % our)
 
         if os.name in ('nt', 'ce'):
             log_handle.write(UPGRADE_WINDOWS)
         else:
             log_handle.write(UPGRADE_OTHER)
 
         sys.exit(1)