Bug 1388013 - Support running |mach python-test| against Python 3 using pipenv; r=ahal
☠☠ backed out by ba9528133000 ☠ ☠
authorDave Hunt <dhunt@mozilla.com>
Thu, 03 May 2018 10:34:22 +0100
changeset 420640 c55bfefbd4e1786bd90ed173bbef547dff8d84ef
parent 420639 3a163da2b21b5b835c6a0e41cd0d35425cc84edb
child 420641 eea857170a418231418d27bda675446e2d21b2b8
push id34076
push usernerli@mozilla.com
push dateThu, 31 May 2018 21:50:41 +0000
treeherdermozilla-central@0dc7bfc7b0c1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1388013
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1388013 - Support running |mach python-test| against Python 3 using pipenv; r=ahal This patch allows executing |mach python-test| against Python 3 by specifying the optional |--three| command line option. When this option is present, pipenv will be used to manage a virtual environment using Python 3 and attempt to run the tests. When it is not present, pipenv will not be used, and everything will work as it did before this patch. My original plan was to use pipenv regardless of the target version of Python, however I encountered several issues running some of the tests against our Python packages. Once all tests have been patched to run against Python 3, then we should be able to use pipenv when running them against Python 2. Note that this patch allows tests to run against Python 3, but there are plenty of issues preventing them from passing. With this patch in place we can start to add Python 3 support to our packages and have the tests running in CI to ensure we don't regress back to just supporting Python 2. MozReview-Commit-ID: BuU5gZK83hL IHG: changed taskcluster/ci/source-test/python.yml
python/Pipfile
python/Pipfile.lock
python/mach/setup.py
python/mach_commands.py
python/mozbuild/mozbuild/base.py
python/mozbuild/mozbuild/virtualenv.py
python/mozversioncontrol/setup.py
new file mode 100644
--- /dev/null
+++ b/python/Pipfile
@@ -0,0 +1,36 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+"d5b4a14" = {path = "./mach"}
+"8ddb376" = {path = "./mozbuild"}
+"b3ddbcf" = {path = "./mozterm"}
+"38a4a9a" = {path = "./mozversioncontrol"}
+"26d92fb" = {path = "./../config/mozunit"}
+"cea2946" = {path = "./../testing/mozbase/manifestparser"}
+"ffcf6e6" = {path = "./../testing/mozbase/mozcrash"}
+"195ae2e" = {path = "./../testing/mozbase/mozdebug"}
+"8dab59a" = {path = "./../testing/mozbase/mozdevice"}
+"58d0848" = {path = "./../testing/mozbase/mozfile"}
+"fd0b608" = {path = "./../testing/mozbase/mozhttpd"}
+"7329809" = {path = "./../testing/mozbase/mozinfo"}
+"501835d" = {path = "./../testing/mozbase/mozinstall"}
+"807c1c5" = {path = "./../testing/mozbase/mozlog"}
+"e09e103" = {path = "./../testing/mozbase/moznetwork"}
+"132adec" = {path = "./../testing/mozbase/mozprocess"}
+"d88f467" = {path = "./../testing/mozbase/mozprofile"}
+"1de94f2" = {path = "./../testing/mozbase/mozrunner"}
+"6477f20" = {path = "./../testing/mozbase/moztest"}
+"f1d74ca" = {path = "./../testing/mozbase/mozversion"}
+"47200d8" = {path = "./../third_party/python/futures", markers="python_version < '3'"}
+"110bcc4" = {path = "./../third_party/python/jsmin"}
+"c49d32a" = {path = "./../third_party/python/mock-1.0.0", markers="python_version < '3.3'"}
+"c2c21d9" = {path = "./../third_party/python/py"}
+"f4b00e9" = {path = "./../third_party/python/pytest"}
+"053111f" = {path = "./../third_party/python/requests"}
+"d250320" = {path = "./../third_party/python/six"}
+"f1de77a" = {path = "./../third_party/python/which", markers="python_version < '3.3'"}
+
+[dev-packages]
new file mode 100644
--- /dev/null
+++ b/python/Pipfile.lock
@@ -0,0 +1,106 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "dfc219f64edc7715acdb35e03dcee665ec26908c18a58d3a3a88dda3ab393b17"
+        },
+        "pipfile-spec": 6,
+        "requires": {},
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "053111f": {
+            "path": "./../third_party/python/requests"
+        },
+        "110bcc4": {
+            "path": "./../third_party/python/jsmin"
+        },
+        "132adec": {
+            "path": "./../testing/mozbase/mozprocess"
+        },
+        "195ae2e": {
+            "path": "./../testing/mozbase/mozdebug"
+        },
+        "1de94f2": {
+            "path": "./../testing/mozbase/mozrunner"
+        },
+        "26d92fb": {
+            "path": "./../config/mozunit"
+        },
+        "38a4a9a": {
+            "path": "./mozversioncontrol"
+        },
+        "47200d8": {
+            "markers": "python_version < '3'",
+            "path": "./../third_party/python/futures"
+        },
+        "501835d": {
+            "path": "./../testing/mozbase/mozinstall"
+        },
+        "58d0848": {
+            "path": "./../testing/mozbase/mozfile"
+        },
+        "6477f20": {
+            "path": "./../testing/mozbase/moztest"
+        },
+        "7329809": {
+            "path": "./../testing/mozbase/mozinfo"
+        },
+        "807c1c5": {
+            "path": "./../testing/mozbase/mozlog"
+        },
+        "8dab59a": {
+            "path": "./../testing/mozbase/mozdevice"
+        },
+        "8ddb376": {
+            "path": "./mozbuild"
+        },
+        "b3ddbcf": {
+            "path": "./mozterm"
+        },
+        "c2c21d9": {
+            "path": "./../third_party/python/py"
+        },
+        "c49d32a": {
+            "markers": "python_version < '3.3'",
+            "path": "./../third_party/python/mock-1.0.0"
+        },
+        "cea2946": {
+            "path": "./../testing/mozbase/manifestparser"
+        },
+        "d250320": {
+            "path": "./../third_party/python/six"
+        },
+        "d5b4a14": {
+            "path": "./mach"
+        },
+        "d88f467": {
+            "path": "./../testing/mozbase/mozprofile"
+        },
+        "e09e103": {
+            "path": "./../testing/mozbase/moznetwork"
+        },
+        "f1d74ca": {
+            "path": "./../testing/mozbase/mozversion"
+        },
+        "f1de77a": {
+            "markers": "python_version < '3.3'",
+            "path": "./../third_party/python/which"
+        },
+        "f4b00e9": {
+            "path": "./../third_party/python/pytest"
+        },
+        "fd0b608": {
+            "path": "./../testing/mozbase/mozhttpd"
+        },
+        "ffcf6e6": {
+            "path": "./../testing/mozbase/mozcrash"
+        }
+    },
+    "develop": {}
+}
--- a/python/mach/setup.py
+++ b/python/mach/setup.py
@@ -15,17 +15,17 @@ README = open('README.rst').read()
 setup(
     name='mach',
     description='Generic command line command dispatching framework.',
     long_description=README,
     license='MPL 2.0',
     author='Gregory Szorc',
     author_email='gregory.szorc@gmail.com',
     url='https://developer.mozilla.org/en-US/docs/Developer_Guide/mach',
-    packages=['mach'],
+    packages=['mach', 'mach.mixin'],
     version=VERSION,
     classifiers=[
         'Environment :: Console',
         'Development Status :: 3 - Alpha',
         'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
         'Natural Language :: English',
     ],
     install_requires=[
--- a/python/mach_commands.py
+++ b/python/mach_commands.py
@@ -25,16 +25,18 @@ from mozbuild.base import (
 )
 
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
 )
 
+here = os.path.abspath(os.path.dirname(__file__))
+
 
 @CommandProvider
 class MachCommands(MachCommandBase):
     @Command('python', category='devenv',
              description='Run Python.')
     @CommandArgument('--no-virtualenv', action='store_true',
                      help='Do not set up a virtualenv')
     @CommandArgument('args', nargs=argparse.REMAINDER)
@@ -60,16 +62,20 @@ class MachCommands(MachCommandBase):
                                 append_env=append_env)
 
     @Command('python-test', category='testing',
              description='Run Python unit tests with an appropriate test runner.')
     @CommandArgument('-v', '--verbose',
                      default=False,
                      action='store_true',
                      help='Verbose output.')
+    @CommandArgument('--three',
+                     default=False,
+                     action='store_true',
+                     help='Run tests using Python 3.')
     @CommandArgument('-j', '--jobs',
                      default=1,
                      type=int,
                      help='Number of concurrent jobs to run. Default is 1.')
     @CommandArgument('--subsuite',
                      default=None,
                      help=('Python subsuite to run. If not specified, all subsuites are run. '
                            'Use the string `default` to only run tests without a subsuite.'))
@@ -86,18 +92,23 @@ class MachCommands(MachCommandBase):
             mozfile.remove(tempdir)
 
     def run_python_tests(self,
                          tests=None,
                          test_objects=None,
                          subsuite=None,
                          verbose=False,
                          jobs=1,
+                         three=False,
                          **kwargs):
-        self._activate_virtualenv()
+        if three:
+            # use pipenv to run tests against Python 3
+            self.activate_pipenv(os.path.join(here, 'Pipfile'), ['--three'])
+        else:
+            self._activate_virtualenv()
 
         if test_objects is None:
             from moztest.resolve import TestResolver
             resolver = self._spawn(TestResolver)
             # If we were given test paths, try to find tests matching them.
             test_objects = resolver.resolve_tests(paths=tests, flavor='python')
         else:
             # We've received test_objects from |mach test|. We need to ignore
--- a/python/mozbuild/mozbuild/base.py
+++ b/python/mozbuild/mozbuild/base.py
@@ -752,21 +752,21 @@ class MozbuildObject(ProcessExecutionMix
         self._activate_virtualenv()
         pipenv = os.path.join(self.virtualenv_manager.bin_path, 'pipenv')
         if not os.path.exists(pipenv):
             pipenv_reqs = os.path.join(self.topsrcdir, 'python/mozbuild/mozbuild/pipenv.txt')
             self.virtualenv_manager.install_pip_requirements(
                 pipenv_reqs, require_hashes=False, vendored=True)
         return pipenv
 
-    def activate_pipenv(self, path):
+    def activate_pipenv(self, path, args=None):
         if not os.path.exists(path):
             raise Exception('Pipfile not found: %s.' % path)
         self.ensure_pipenv()
-        self.virtualenv_manager.activate_pipenv(path)
+        self.virtualenv_manager.activate_pipenv(path, args)
 
 
 class MachCommandBase(MozbuildObject):
     """Base class for mach command providers that wish to be MozbuildObjects.
 
     This provides a level of indirection so MozbuildObject can be refactored
     without having to change everything that inherits from it.
     """
--- a/python/mozbuild/mozbuild/virtualenv.py
+++ b/python/mozbuild/mozbuild/virtualenv.py
@@ -528,28 +528,29 @@ class VirtualenvManager(object):
         # This will confuse pip and cause the package to attempt to install
         # against the executing interpreter. By creating a new process, we
         # force the virtualenv's interpreter to be used and all is well.
         # It /might/ be possible to cheat and set sys.executable to
         # self.python_path. However, this seems more risk than it's worth.
         subprocess.check_call([os.path.join(self.bin_path, 'pip')] + args,
             stderr=subprocess.STDOUT)
 
-    def activate_pipenv(self, pipfile):
+    def activate_pipenv(self, pipfile, args=None):
         """Install a Pipfile located at path and activate environment"""
         pipenv = os.path.join(self.bin_path, 'pipenv')
         env = os.environ.copy()
         env.update({
             'PIPENV_IGNORE_VIRTUALENVS': '1',
             'PIPENV_PIPFILE': pipfile,
             'WORKON_HOME': os.path.join(self.topobjdir, '_virtualenvs'),
         })
 
+        args = args or []
         subprocess.check_call(
-            [pipenv, 'install', '--deploy'],
+            [pipenv, 'install'] + args,
             stderr=subprocess.STDOUT,
             env=env)
 
         self.virtualenv_root = subprocess.check_output(
             [pipenv, '--venv'],
             stderr=subprocess.STDOUT,
             env=env).rstrip()
 
new file mode 100644
--- /dev/null
+++ b/python/mozversioncontrol/setup.py
@@ -0,0 +1,27 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+from setuptools import setup, find_packages
+
+VERSION = '0.1'
+
+setup(
+    author='Mozilla Foundation',
+    author_email='Mozilla Release Engineering',
+    name='mozversioncontrol',
+    description='Mozilla version control functionality',
+    license='MPL 2.0',
+    packages=find_packages(),
+    version=VERSION,
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Topic :: Software Development :: Build Tools',
+        'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: Implementation :: CPython',
+    ],
+    keywords='mozilla',
+)