Bug 1466211 - Switch all |mach python-test| tests to run using pipenv; r=ahal
☠☠ backed out by bfde37aecadd ☠ ☠
authorDave Hunt <dhunt@mozilla.com>
Fri, 08 Jun 2018 13:24:27 +0100
changeset 477568 23eb377e2e6d12ecf1ceb2c8e58fc41ccf89896f
parent 477567 fe821d61ecbfcaf68ae180258249b3b9434213fe
child 477569 dfdf4ea83655c4663c5b75f5d096e5ebb75f7fed
push id9386
push usernbeleuzu@mozilla.com
push dateSat, 23 Jun 2018 22:41:50 +0000
treeherdermozilla-beta@c73f274c9f25 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1466211
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1466211 - Switch all |mach python-test| tests to run using pipenv; r=ahal MozReview-Commit-ID: AzmdDgAgZgI
Pipfile
Pipfile.lock
python/Pipfile
python/Pipfile.lock
python/mach_commands.py
python/mozbuild/mozbuild/base.py
python/mozbuild/mozbuild/virtualenv.py
--- a/Pipfile
+++ b/Pipfile
@@ -11,12 +11,8 @@ jsmin = "==2.1.0"
 json-e = "==2.5.0"
 pipenv = "==2018.5.18"
 pytest = "==3.2.5"
 python-hglib = "==2.4"
 requests = "==2.9.1"
 six = "==1.10.0"
 virtualenv = "==15.2.0"
 voluptuous = "==0.10.5"
-
-
-[requires]
-python_version = "2.7"
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,17 +1,15 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "af4e239c88ce3d74e2e3dd7d352c3e8a203ce476c7369b2a4dc0eea7114996ba"
+            "sha256": "eb8b0a9771d4420f83fbbabf9952dc783aeefe9be455559de2f3ebff27caa93f"
         },
         "pipfile-spec": 6,
-        "requires": {
-            "python_version": "2.7"
-        },
+        "requires": {},
         "sources": [
             {
                 "name": "pypi",
                 "url": "https://pypi.org/simple",
                 "verify_ssl": true
             }
         ]
     },
deleted file mode 100644
--- a/python/Pipfile
+++ /dev/null
@@ -1,36 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-"d5b4a14" = {path = "./mach"}
-"8ddb376" = {path = "./mozbuild"}
-"b3ddbcf" = {path = "./mozterm"}
-"38a4a9a" = {path = "./mozversioncontrol"}
-"26d92fb" = {path = "./../config/mozunit"}
-"cea2946" = {path = "./../testing/mozbase/manifestparser"}
-"ffcf6e6" = {path = "./../testing/mozbase/mozcrash"}
-"195ae2e" = {path = "./../testing/mozbase/mozdebug"}
-"8dab59a" = {path = "./../testing/mozbase/mozdevice"}
-"58d0848" = {path = "./../testing/mozbase/mozfile"}
-"fd0b608" = {path = "./../testing/mozbase/mozhttpd"}
-"7329809" = {path = "./../testing/mozbase/mozinfo"}
-"501835d" = {path = "./../testing/mozbase/mozinstall"}
-"807c1c5" = {path = "./../testing/mozbase/mozlog"}
-"e09e103" = {path = "./../testing/mozbase/moznetwork"}
-"132adec" = {path = "./../testing/mozbase/mozprocess"}
-"d88f467" = {path = "./../testing/mozbase/mozprofile"}
-"1de94f2" = {path = "./../testing/mozbase/mozrunner"}
-"6477f20" = {path = "./../testing/mozbase/moztest"}
-"f1d74ca" = {path = "./../testing/mozbase/mozversion"}
-"47200d8" = {path = "./../third_party/python/futures", markers="python_version < '3'"}
-"110bcc4" = {path = "./../third_party/python/jsmin"}
-"c49d32a" = {path = "./../third_party/python/mock-1.0.0", markers="python_version < '3.3'"}
-"c2c21d9" = {path = "./../third_party/python/py"}
-"f4b00e9" = {path = "./../third_party/python/pytest"}
-"053111f" = {path = "./../third_party/python/requests"}
-"d250320" = {path = "./../third_party/python/six"}
-"f1de77a" = {path = "./../third_party/python/which", markers="python_version < '3.3'"}
-
-[dev-packages]
deleted file mode 100644
--- a/python/Pipfile.lock
+++ /dev/null
@@ -1,106 +0,0 @@
-{
-    "_meta": {
-        "hash": {
-            "sha256": "dfc219f64edc7715acdb35e03dcee665ec26908c18a58d3a3a88dda3ab393b17"
-        },
-        "pipfile-spec": 6,
-        "requires": {},
-        "sources": [
-            {
-                "name": "pypi",
-                "url": "https://pypi.org/simple",
-                "verify_ssl": true
-            }
-        ]
-    },
-    "default": {
-        "053111f": {
-            "path": "./../third_party/python/requests"
-        },
-        "110bcc4": {
-            "path": "./../third_party/python/jsmin"
-        },
-        "132adec": {
-            "path": "./../testing/mozbase/mozprocess"
-        },
-        "195ae2e": {
-            "path": "./../testing/mozbase/mozdebug"
-        },
-        "1de94f2": {
-            "path": "./../testing/mozbase/mozrunner"
-        },
-        "26d92fb": {
-            "path": "./../config/mozunit"
-        },
-        "38a4a9a": {
-            "path": "./mozversioncontrol"
-        },
-        "47200d8": {
-            "markers": "python_version < '3'",
-            "path": "./../third_party/python/futures"
-        },
-        "501835d": {
-            "path": "./../testing/mozbase/mozinstall"
-        },
-        "58d0848": {
-            "path": "./../testing/mozbase/mozfile"
-        },
-        "6477f20": {
-            "path": "./../testing/mozbase/moztest"
-        },
-        "7329809": {
-            "path": "./../testing/mozbase/mozinfo"
-        },
-        "807c1c5": {
-            "path": "./../testing/mozbase/mozlog"
-        },
-        "8dab59a": {
-            "path": "./../testing/mozbase/mozdevice"
-        },
-        "8ddb376": {
-            "path": "./mozbuild"
-        },
-        "b3ddbcf": {
-            "path": "./mozterm"
-        },
-        "c2c21d9": {
-            "path": "./../third_party/python/py"
-        },
-        "c49d32a": {
-            "markers": "python_version < '3.3'",
-            "path": "./../third_party/python/mock-1.0.0"
-        },
-        "cea2946": {
-            "path": "./../testing/mozbase/manifestparser"
-        },
-        "d250320": {
-            "path": "./../third_party/python/six"
-        },
-        "d5b4a14": {
-            "path": "./mach"
-        },
-        "d88f467": {
-            "path": "./../testing/mozbase/mozprofile"
-        },
-        "e09e103": {
-            "path": "./../testing/mozbase/moznetwork"
-        },
-        "f1d74ca": {
-            "path": "./../testing/mozbase/mozversion"
-        },
-        "f1de77a": {
-            "markers": "python_version < '3.3'",
-            "path": "./../third_party/python/which"
-        },
-        "f4b00e9": {
-            "path": "./../third_party/python/pytest"
-        },
-        "fd0b608": {
-            "path": "./../testing/mozbase/mozhttpd"
-        },
-        "ffcf6e6": {
-            "path": "./../testing/mozbase/mozcrash"
-        }
-    },
-    "develop": {}
-}
--- a/python/mach_commands.py
+++ b/python/mach_commands.py
@@ -94,21 +94,18 @@ class MachCommands(MachCommandBase):
     def run_python_tests(self,
                          tests=None,
                          test_objects=None,
                          subsuite=None,
                          verbose=False,
                          jobs=1,
                          three=False,
                          **kwargs):
-        if three:
-            # use pipenv to run tests against Python 3
-            self.activate_pipenv(os.path.join(here, 'Pipfile'), ['--three'])
-        else:
-            self._activate_virtualenv()
+        pipenv_args = ['--three' if three else '--two']
+        self.activate_pipenv(pipfile=None, args=pipenv_args, populate=True)
 
         if test_objects is None:
             from moztest.resolve import TestResolver
             resolver = self._spawn(TestResolver)
             # If we were given test paths, try to find tests matching them.
             test_objects = resolver.resolve_tests(paths=tests, flavor='python')
         else:
             # We've received test_objects from |mach test|. We need to ignore
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -752,21 +752,21 @@ class MozbuildObject(ProcessExecutionMix
         self._activate_virtualenv()
         pipenv = os.path.join(self.virtualenv_manager.bin_path, 'pipenv')
         if not os.path.exists(pipenv):
             pipenv_reqs = os.path.join(self.topsrcdir, 'python/mozbuild/mozbuild/pipenv.txt')
             self.virtualenv_manager.install_pip_requirements(
                 pipenv_reqs, require_hashes=False, vendored=True)
         return pipenv
 
-    def activate_pipenv(self, path, args=None):
-        if not os.path.exists(path):
-            raise Exception('Pipfile not found: %s.' % path)
+    def activate_pipenv(self, pipfile=None, args=None, populate=False):
+        if pipfile is not None and not os.path.exists(pipfile):
+            raise Exception('Pipfile not found: %s.' % pipfile)
         self.ensure_pipenv()
-        self.virtualenv_manager.activate_pipenv(path, args)
+        self.virtualenv_manager.activate_pipenv(pipfile, args, populate)
 
 
 class MachCommandBase(MozbuildObject):
     """Base class for mach command providers that wish to be MozbuildObjects.
 
     This provides a level of indirection so MozbuildObject can be refactored
     without having to change everything that inherits from it.
     """
--- a/python/mozbuild/mozbuild/virtualenv.py
+++ b/python/mozbuild/mozbuild/virtualenv.py
@@ -15,34 +15,38 @@ import sys
 import warnings
 
 from distutils.version import LooseVersion
 
 IS_NATIVE_WIN = (sys.platform == 'win32' and os.sep == '\\')
 IS_MSYS2 = (sys.platform == 'win32' and os.sep == '/')
 IS_CYGWIN = (sys.platform == 'cygwin')
 
-# Minimum version of Python required to build.
-MINIMUM_PYTHON_VERSION = LooseVersion('2.7.3')
-MINIMUM_PYTHON_MAJOR = 2
+# Minimum versions of Python required to build.
+MINIMUM_PYTHON_VERSIONS = {
+    2: LooseVersion('2.7.3'),
+    3: LooseVersion('3.5.0')
+}
 
 
 UPGRADE_WINDOWS = '''
 Please upgrade to the latest MozillaBuild development environment. See
 https://developer.mozilla.org/en-US/docs/Developer_Guide/Build_Instructions/Windows_Prerequisites
 '''.lstrip()
 
 UPGRADE_OTHER = '''
 Run |mach bootstrap| to ensure your system is up to date.
 
 If you still receive this error, your shell environment is likely detecting
 another Python version. Ensure a modern Python can be found in the paths
 defined by the $PATH environment variable and try again.
 '''.lstrip()
 
+here = os.path.abspath(os.path.dirname(__file__))
+
 
 class VirtualenvManager(object):
     """Contains logic for managing virtualenvs for building the tree."""
 
     def __init__(self, topsrcdir, topobjdir, virtualenv_path, log_handle,
         manifest_path):
         """Create a new manager.
 
@@ -202,17 +206,17 @@ class VirtualenvManager(object):
             raise Exception(
                 'Failed to create virtualenv: %s' % self.virtualenv_root)
 
         self.write_exe_info(python)
 
         return self.virtualenv_root
 
     def packages(self):
-        with file(self.manifest_path, 'rU') as fh:
+        with open(self.manifest_path, 'rU') as fh:
             packages = [line.rstrip().split(':')
                         for line in fh]
         return packages
 
     def populate(self):
         """Populate the virtualenv.
 
         The manifest file consists of colon-delimited fields. The first field
@@ -245,17 +249,16 @@ class VirtualenvManager(object):
             search path. e.g. "objdir:build" will add $topobjdir/build to the
             search path.
 
         Note that the Python interpreter running this function should be the
         one from the virtualenv. If it is the system Python or if the
         environment is not configured properly, packages could be installed
         into the wrong place. This is how virtualenv's work.
         """
-
         packages = self.packages()
         python_lib = distutils.sysconfig.get_python_lib()
 
         def handle_package(package):
             if package[0] == 'setup.py':
                 assert len(package) >= 2
 
                 self.call_setup(os.path.join(self.topsrcdir, package[1]),
@@ -525,52 +528,72 @@ class VirtualenvManager(object):
     def _run_pip(self, args):
         # It's tempting to call pip natively via pip.main(). However,
         # the current Python interpreter may not be the virtualenv python.
         # This will confuse pip and cause the package to attempt to install
         # against the executing interpreter. By creating a new process, we
         # force the virtualenv's interpreter to be used and all is well.
         # It /might/ be possible to cheat and set sys.executable to
         # self.python_path. However, this seems more risk than it's worth.
-        subprocess.check_call([os.path.join(self.bin_path, 'pip')] + args,
-            stderr=subprocess.STDOUT)
+        pip = os.path.join(self.bin_path, 'pip')
+        subprocess.check_call([pip] + args, stderr=subprocess.STDOUT, cwd=self.topsrcdir)
+
+    def activate_pipenv(self, pipfile=None, args=None, populate=False):
+        """Activate a virtual environment managed by pipenv
 
-    def activate_pipenv(self, pipfile, args=None):
-        """Install a Pipfile located at path and activate environment"""
+        If ``pipfile`` is not ``None`` then the Pipfile located at the path
+        provided will be used to create the virtual environment. If
+        ``populate`` is ``True`` then the virtual environment will be
+        populated from the manifest file. The optional ``args`` list will be
+        passed to the pipenv commands.
+        """
         pipenv = os.path.join(self.bin_path, 'pipenv')
         env = os.environ.copy()
         env.update({
             'PIPENV_IGNORE_VIRTUALENVS': '1',
-            'PIPENV_PIPFILE': pipfile,
             'WORKON_HOME': os.path.join(self.topobjdir, '_virtualenvs'),
         })
 
         args = args or []
+
+        if pipfile is not None:
+            # Install from Pipfile
+            env['PIPENV_PIPFILE'] = pipfile
+            args.append('install')
+
         subprocess.check_call(
-            [pipenv, 'install'] + args,
+            [pipenv] + args,
             stderr=subprocess.STDOUT,
             env=env)
 
         self.virtualenv_root = subprocess.check_output(
             [pipenv, '--venv'],
             stderr=subprocess.STDOUT,
             env=env).rstrip()
 
+        if populate:
+            # Populate from the manifest
+            subprocess.check_call([
+                pipenv, 'run', 'python', os.path.join(here, 'virtualenv.py'), 'populate',
+                self.topsrcdir, self.topobjdir, self.virtualenv_root, self.manifest_path],
+                stderr=subprocess.STDOUT, env=env)
+
         self.activate()
 
 
 def verify_python_version(log_handle):
     """Ensure the current version of Python is sufficient."""
     major, minor, micro = sys.version_info[:3]
 
     our = LooseVersion('%d.%d.%d' % (major, minor, micro))
 
-    if major != MINIMUM_PYTHON_MAJOR or our < MINIMUM_PYTHON_VERSION:
-        log_handle.write('Python %s or greater (but not Python 3) is '
-            'required to build. ' % MINIMUM_PYTHON_VERSION)
+    if major not in MINIMUM_PYTHON_VERSIONS or our < MINIMUM_PYTHON_VERSIONS[major]:
+        log_handle.write('The following Python versions are required to build:\n')
+        for minver in MINIMUM_PYTHON_VERSIONS.values():
+            log_handle.write('* Python %s or greater\n' % minver)
         log_handle.write('You are running Python %s.\n' % our)
 
         if os.name in ('nt', 'ce'):
             log_handle.write(UPGRADE_WINDOWS)
         else:
             log_handle.write(UPGRADE_OTHER)
 
         sys.exit(1)