Bug 1712151: Add test to verify virtualenv compatibility r=ahal
☠☠ backed out by b59ab731aecf ☠ ☠
authorMitchell Hentges <mhentges@mozilla.com>
Mon, 27 Sep 2021 20:27:19 +0000
changeset 593334 7adf7e2136a3bf870571eb22b6f4200f907124e3
parent 593333 2aef162b9a1b6927c62b01e3e9c60ac1676af36d
child 593335 6e1bde40e3b46701299107ac2e22fd43e48f475e
push id38827
push userctuns@mozilla.com
push dateTue, 28 Sep 2021 09:54:33 +0000
treeherdermozilla-central@c985110bb4b5 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1712151
milestone94.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1712151: Add test to verify virtualenv compatibility r=ahal This adds two main compatibility guarantees: 1. Vendored dependencies <=> Pypi-downloaded dependencies 2. Global Mach dependencies <=> command-specific dependencies As part of this, a new `vendored:` action was added to the virtualenv definition format. Otherwise similar to `pth:` paths, `vendored:` packages are assumed to be "pip install"-able. Some validation (the `.dist-info`/`PKG-INFO` checks) was added to `requirements.py` to verify that `pth:` and `vendored:` are correctly used. Differential Revision: https://phabricator.services.mozilla.com/D122900
build/build_virtualenv_packages.txt
build/common_virtualenv_packages.txt
build/mach_initialize.py
build/python-test_virtualenv_packages.txt
python/mach/mach/requirements.py
python/mach/mach/test/python.ini
python/mach/mach/test/test_virtualenv_compatibility.py
python/mach_commands.py
python/mozbuild/mozbuild/test/test_vendor.py
python/mozbuild/mozbuild/virtualenv.py
--- a/build/build_virtualenv_packages.txt
+++ b/build/build_virtualenv_packages.txt
@@ -1,2 +1,2 @@
 packages.txt:build/common_virtualenv_packages.txt
-pth:third_party/python/glean_parser
+vendored:third_party/python/glean_parser
--- a/build/common_virtualenv_packages.txt
+++ b/build/common_virtualenv_packages.txt
@@ -39,89 +39,87 @@ pth:testing/mozbase/mozproxy
 pth:testing/mozbase/mozrunner
 pth:testing/mozbase/mozsystemmonitor
 pth:testing/mozbase/mozscreenshot
 pth:testing/mozbase/moztest
 pth:testing/mozbase/mozversion
 pth:testing/raptor
 pth:testing/talos
 pth:testing/web-platform
-pth:testing/web-platform/tests/tools/third_party/certifi
-pth:testing/web-platform/tests/tools/third_party/h2
-pth:testing/web-platform/tests/tools/third_party/hpack
-pth:testing/web-platform/tests/tools/third_party/html5lib
-pth:testing/web-platform/tests/tools/third_party/hyperframe
-pth:testing/web-platform/tests/tools/third_party/pywebsocket3
-pth:testing/web-platform/tests/tools/third_party/webencodings
-pth:testing/web-platform/tests/tools/wptserve
-pth:testing/web-platform/tests/tools/wptrunner
+vendored:testing/web-platform/tests/tools/third_party/certifi
+vendored:testing/web-platform/tests/tools/third_party/h2
+vendored:testing/web-platform/tests/tools/third_party/hpack
+vendored:testing/web-platform/tests/tools/third_party/html5lib
+vendored:testing/web-platform/tests/tools/third_party/hyperframe
+vendored:testing/web-platform/tests/tools/third_party/pywebsocket3
+vendored:testing/web-platform/tests/tools/third_party/webencodings
+vendored:testing/web-platform/tests/tools/wptserve
+vendored:testing/web-platform/tests/tools/wptrunner
 pth:testing/xpcshell
-pth:third_party/python/aiohttp
-pth:third_party/python/appdirs
-pth:third_party/python/async_timeout
-pth:third_party/python/atomicwrites
-pth:third_party/python/attrs
-pth:third_party/python/blessings
-pth:third_party/python/cbor2
-pth:third_party/python/chardet
-pth:third_party/python/Click
-pth:third_party/python/compare_locales
-pth:third_party/python/cookies
-pth:third_party/python/cram
-pth:third_party/python/diskcache
-pth:third_party/python/distro
-pth:third_party/python/dlmanager
-pth:third_party/python/ecdsa
-pth:third_party/python/esprima
-pth:third_party/python/fluent.migrate
-pth:third_party/python/fluent.syntax
-pth:third_party/python/funcsigs
-pth:third_party/python/gyp/pylib
-pth:third_party/python/idna
-pth:third_party/python/idna-ssl
-pth:third_party/python/importlib_metadata
-pth:third_party/python/iso8601
-pth:third_party/python/Jinja2
-pth:third_party/python/jsmin
-pth:third_party/python/json-e
-pth:third_party/python/jsonschema
-pth:third_party/python/MarkupSafe/src
-pth:third_party/python/mohawk
-pth:third_party/python/more_itertools
-pth:third_party/python/mozilla_version
-pth:third_party/python/multidict
-pth:third_party/python/packaging
-pth:third_party/python/pathspec
-pth:third_party/python/pip_tools
-pth:third_party/python/pluggy
-pth:third_party/python/ply
-pth:third_party/python/py
-pth:third_party/python/pyasn1
-pth:third_party/python/pyasn1_modules
-pth:third_party/python/pylru
-pth:third_party/python/pyparsing
-pth:third_party/python/pyrsistent
-pth:third_party/python/pystache
-pth:third_party/python/pytest
-pth:third_party/python/python-hglib
-pth:third_party/python/pytoml
-pth:third_party/python/PyYAML/lib3/
-pth:third_party/python/redo
-pth:third_party/python/requests
-pth:third_party/python/requests_unixsocket
-pth:third_party/python/responses
-pth:third_party/python/rsa
-pth:third_party/python/sentry_sdk
-pth:third_party/python/six
-pth:third_party/python/slugid
-pth:third_party/python/taskcluster
-pth:third_party/python/taskcluster_urls
-pth:third_party/python/typing_extensions
-pth:third_party/python/urllib3
-pth:third_party/python/voluptuous
-pth:third_party/python/yamllint
-pth:third_party/python/yarl
-pth:third_party/python/zipp
+vendored:third_party/python/aiohttp
+vendored:third_party/python/appdirs
+vendored:third_party/python/async_timeout
+vendored:third_party/python/atomicwrites
+vendored:third_party/python/attrs
+vendored:third_party/python/blessings
+vendored:third_party/python/cbor2
+vendored:third_party/python/chardet
+vendored:third_party/python/Click
+vendored:third_party/python/compare_locales
+vendored:third_party/python/cookies
+vendored:third_party/python/cram
+vendored:third_party/python/diskcache
+vendored:third_party/python/distro
+vendored:third_party/python/dlmanager
+vendored:third_party/python/ecdsa
+vendored:third_party/python/esprima
+vendored:third_party/python/fluent.migrate
+vendored:third_party/python/fluent.syntax
+vendored:third_party/python/funcsigs
+vendored:third_party/python/gyp/pylib
+vendored:third_party/python/idna
+vendored:third_party/python/idna-ssl
+vendored:third_party/python/importlib_metadata
+vendored:third_party/python/iso8601
+vendored:third_party/python/Jinja2
+vendored:third_party/python/jsmin
+vendored:third_party/python/json-e
+vendored:third_party/python/jsonschema
+vendored:third_party/python/MarkupSafe/src
+vendored:third_party/python/mohawk
+vendored:third_party/python/more_itertools
+vendored:third_party/python/mozilla_version
+vendored:third_party/python/multidict
+vendored:third_party/python/pathspec
+vendored:third_party/python/pip_tools
+vendored:third_party/python/pluggy
+vendored:third_party/python/ply
+vendored:third_party/python/py
+vendored:third_party/python/pyasn1
+vendored:third_party/python/pyasn1_modules
+vendored:third_party/python/pylru
+vendored:third_party/python/pyrsistent
+vendored:third_party/python/pystache
+vendored:third_party/python/pytest
+vendored:third_party/python/python-hglib
+vendored:third_party/python/pytoml
+vendored:third_party/python/PyYAML/lib3/
+vendored:third_party/python/redo
+vendored:third_party/python/requests
+vendored:third_party/python/requests_unixsocket
+vendored:third_party/python/responses
+vendored:third_party/python/rsa
+vendored:third_party/python/sentry_sdk
+vendored:third_party/python/six
+vendored:third_party/python/slugid
+vendored:third_party/python/taskcluster
+vendored:third_party/python/taskcluster_urls
+vendored:third_party/python/typing_extensions
+vendored:third_party/python/urllib3
+vendored:third_party/python/voluptuous
+vendored:third_party/python/yamllint
+vendored:third_party/python/yarl
+vendored:third_party/python/zipp
 pth:toolkit/components/telemetry/tests/marionette/harness
 pth:tools
 pth:tools/moztreedocs
 pth:xpcom/ds/tools
 pth:xpcom/idl-parser
--- a/build/mach_initialize.py
+++ b/build/mach_initialize.py
@@ -180,17 +180,18 @@ def _activate_python_environment(topsrcd
     )
 
     requirements = MachEnvRequirements.from_requirements_definition(
         topsrcdir,
         is_thunderbird,
         os.path.join(topsrcdir, "build", "mach_virtualenv_packages.txt"),
     )
     sys.path[0:0] = [
-        os.path.join(topsrcdir, pth.path) for pth in requirements.pth_requirements
+        os.path.join(topsrcdir, pth.path)
+        for pth in requirements.pth_requirements + requirements.vendored_requirements
     ]
 
 
 def initialize(topsrcdir):
     # Ensure we are running Python 3.6+. We run this check as soon as
     # possible to avoid a cryptic import/usage error.
     if sys.version_info < (3, 6):
         print("Python 3.6+ is required to run mach.")
--- a/build/python-test_virtualenv_packages.txt
+++ b/build/python-test_virtualenv_packages.txt
@@ -1,2 +1,3 @@
 packages.txt:build/common_virtualenv_packages.txt
-pth:third_party/python/glean_parser
+vendored:third_party/python/glean_parser
+
--- a/python/mach/mach/requirements.py
+++ b/python/mach/mach/requirements.py
@@ -1,13 +1,14 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
+from pathlib import Path
 
 
 THUNDERBIRD_PYPI_ERROR = """
 Thunderbird requirements definitions cannot include PyPI packages.
 """.strip()
 
 
 class PthSpecifier:
@@ -55,39 +56,65 @@ class MachEnvRequirements:
         and they can cannot have "pypi" or "pypi-optional" entries.
     """
 
     def __init__(self):
         self.requirements_paths = []
         self.pth_requirements = []
         self.pypi_requirements = []
         self.pypi_optional_requirements = []
+        self.vendored_requirements = []
 
     @classmethod
     def from_requirements_definition(
         cls, topsrcdir, is_thunderbird, requirements_definition
     ):
         requirements = cls()
         _parse_mach_env_requirements(
             requirements, requirements_definition, topsrcdir, is_thunderbird
         )
         return requirements
 
 
 def _parse_mach_env_requirements(
     requirements_output, root_requirements_path, topsrcdir, is_thunderbird
 ):
-    def _parse_requirements_line(line, is_thunderbird_packages_txt):
+    topsrcdir = Path(topsrcdir)
+
+    def _parse_requirements_line(
+        current_requirements_path, line, line_number, is_thunderbird_packages_txt
+    ):
         line = line.strip()
         if not line or line.startswith("#"):
             return
 
         action, params = line.rstrip().split(":", maxsplit=1)
         if action == "pth":
+            path = topsrcdir / params
+            if not path.exists():
+                # In sparse checkouts, not all paths will be populated.
+                return
+
+            for child in path.iterdir():
+                if child.name.endswith(".dist-info"):
+                    raise Exception(
+                        f'The "pth:" pointing to "{path}" has a ".dist-info" file.\n'
+                        f'Perhaps "{current_requirements_path}:{line_number}" '
+                        'should change to start with "vendored:" instead of "pth:".'
+                    )
+                if child.name == "PKG-INFO":
+                    raise Exception(
+                        f'The "pth:" pointing to "{path}" has a "PKG-INFO" file.\n'
+                        f'Perhaps "{current_requirements_path}:{line_number}" '
+                        'should change to start with "vendored:" instead of "pth:".'
+                    )
+
             requirements_output.pth_requirements.append(PthSpecifier(params))
+        elif action == "vendored":
+            requirements_output.vendored_requirements.append(PthSpecifier(params))
         elif action == "packages.txt":
             _parse_requirements_definition_file(
                 os.path.join(topsrcdir, params),
                 is_thunderbird_packages_txt,
             )
         elif action == "pypi":
             if is_thunderbird_packages_txt:
                 raise Exception(THUNDERBIRD_PYPI_ERROR)
@@ -124,18 +151,20 @@ def _parse_mach_env_requirements(
     ):
         """Parse requirements file into list of requirements"""
         assert os.path.isfile(requirements_path)
         requirements_output.requirements_paths.append(requirements_path)
 
         with open(requirements_path, "r") as requirements_file:
             lines = [line for line in requirements_file]
 
-        for line in lines:
-            _parse_requirements_line(line, is_thunderbird_packages_txt)
+        for number, line in enumerate(lines, start=1):
+            _parse_requirements_line(
+                requirements_path, line, number, is_thunderbird_packages_txt
+            )
 
     _parse_requirements_definition_file(root_requirements_path, False)
 
 
 def _parse_package_specifier(specifier):
     if len(specifier.split("==")) != 2:
         raise Exception(
             "Expected pypi package version to be pinned in the "
--- a/python/mach/mach/test/python.ini
+++ b/python/mach/mach/test/python.ini
@@ -7,8 +7,14 @@ skip-if = python == 3
 [test_config.py]
 [test_decorators.py]
 [test_dispatcher.py]
 [test_entry_point.py]
 [test_error_output.py]
 skip-if = python == 3
 [test_logger.py]
 [test_mach.py]
+[test_virtualenv_compatibility.py]
+# The Windows and Mac workers only use the internal PyPI mirror,
+# which will be missing packages required for this test.
+skip-if =
+  os == "win"
+  os == "mac"
new file mode 100644
--- /dev/null
+++ b/python/mach/mach/test/test_virtualenv_compatibility.py
@@ -0,0 +1,134 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+import mozunit
+from buildconfig import topsrcdir
+from mach.requirements import MachEnvRequirements
+
+
+def _resolve_command_virtualenv_names():
+    virtualenv_names = []
+    for child in (Path(topsrcdir) / "build").iterdir():
+        if not child.name.endswith("_virtualenv_packages.txt"):
+            continue
+
+        if child.name == "mach_virtualenv_packages.txt":
+            continue
+
+        virtualenv_names.append(child.name[: -len("_virtualenv_packages.txt")])
+    return virtualenv_names
+
+
+def _requirement_definition_to_pip_format(virtualenv_name, cache, is_mach_env):
+    """Convert from parsed requirements object to pip-consumable format"""
+    path = Path(topsrcdir) / "build" / f"{virtualenv_name}_virtualenv_packages.txt"
+    requirements = MachEnvRequirements.from_requirements_definition(
+        topsrcdir, False, path
+    )
+
+    lines = []
+    for pypi in (
+        requirements.pypi_requirements + requirements.pypi_optional_requirements
+    ):
+        lines.append(pypi.full_specifier)
+
+    for vendored in requirements.vendored_requirements:
+        lines.append(cache.package_for_vendor_dir(Path(vendored.path)))
+
+    return "\n".join(lines)
+
+
+class PackageCache:
+    def __init__(self, storage_dir: Path):
+        self._cache = {}
+        self._storage_dir = storage_dir
+
+    def package_for_vendor_dir(self, vendor_path: Path):
+        if vendor_path in self._cache:
+            return self._cache[vendor_path]
+
+        if not any((p for p in vendor_path.iterdir() if p.name.endswith(".dist-info"))):
+            # This vendored package is not a wheel. It may be a source package (with
+            # a setup.py), or just some Python code that was manually copied into the
+            # tree. If it's a source package, the setup.py file may be up a few levels
+            # from the referenced Python module path.
+            package_dir = vendor_path
+            while True:
+                if (package_dir / "setup.py").exists():
+                    break
+                elif package_dir.parent == package_dir:
+                    raise Exception(
+                        f'Package "{vendor_path}" is not a wheel and does not have a '
+                        'setup.py file. Perhaps it should be "pth:" instead of '
+                        '"vendored:"?'
+                    )
+                package_dir = package_dir.parent
+
+            self._cache[vendor_path] = str(package_dir)
+            return str(package_dir)
+
+        # Pip requires that wheels have a version number in their name, even if
+        # it ignores it. We should parse out the version and put it in here
+        # so that failure debugging is easier, but that's non-trivial work.
+        # So, this "0" satisfies pip's naming requirement while being relatively
+        # obvious that it's a placeholder.
+        output_path = str(self._storage_dir / f"{vendor_path.name}-0-py3-none-any")
+        shutil.make_archive(output_path, "zip", vendor_path)
+        whl_path = output_path + ".whl"
+        os.rename(output_path + ".zip", whl_path)
+        self._cache[vendor_path] = whl_path
+        return whl_path
+
+
+def test_virtualenvs_compatible(tmpdir):
+    command_virtualenv_names = _resolve_command_virtualenv_names()
+    work_dir = Path(tmpdir)
+    cache = PackageCache(work_dir)
+    mach_requirements = _requirement_definition_to_pip_format("mach", cache, True)
+
+    # Create virtualenv to try to install all dependencies into.
+    subprocess.check_call(
+        [
+            sys.executable,
+            os.path.join(
+                topsrcdir,
+                "third_party",
+                "python",
+                "virtualenv",
+                "virtualenv.py",
+            ),
+            "--no-download",
+            str(work_dir / "env"),
+        ]
+    )
+
+    for name in command_virtualenv_names:
+        print(f'Checking compatibility of "{name}" virtualenv')
+        command_requirements = _requirement_definition_to_pip_format(name, cache, False)
+        with open(work_dir / "requirements.txt", "w") as requirements_txt:
+            requirements_txt.write(mach_requirements)
+            requirements_txt.write("\n")
+            requirements_txt.write(command_requirements)
+
+        # Attempt to install combined set of dependencies (global Mach + current
+        # command)
+        subprocess.check_call(
+            [
+                str(work_dir / "env" / "bin" / "pip"),
+                "install",
+                "-r",
+                str(work_dir / "requirements.txt"),
+            ],
+            cwd=topsrcdir,
+        )
+
+
+if __name__ == "__main__":
+    mozunit.main()
--- a/python/mach_commands.py
+++ b/python/mach_commands.py
@@ -76,16 +76,17 @@ def python(
             os.path.join(
                 command_context.topsrcdir, "build", "mach_virtualenv_packages.txt"
             ),
         )
 
         append_env["PYTHONPATH"] = os.pathsep.join(
             os.path.join(command_context.topsrcdir, pth.path)
             for pth in requirements.pth_requirements
+            + requirements.vendored_requirements
         )
     else:
         command_context.virtualenv_manager.ensure()
         if not no_activate:
             command_context.virtualenv_manager.activate()
         python_path = command_context.virtualenv_manager.python_path
         if requirements:
             command_context.virtualenv_manager.install_pip_requirements(
--- a/python/mozbuild/mozbuild/test/test_vendor.py
+++ b/python/mozbuild/mozbuild/test/test_vendor.py
@@ -24,18 +24,18 @@ def test_up_to_date_vendor():
         # Create empty virtualenv_packages file
         with open(
             os.path.join(work_dir, "build", "common_virtualenv_packages.txt"), "a"
         ) as file:
             # Since VendorPython thinks "work_dir" is the topsrcdir,
             # it will use its associated virtualenv and package configuration.
             # Since it uses "pip-tools" within, and "pip-tools" needs
             # the "Click" library, we need to make them available.
-            file.write("pth:third_party/python/Click\n")
-            file.write("pth:third_party/python/pip_tools\n")
+            file.write("vendored:third_party/python/Click\n")
+            file.write("vendored:third_party/python/pip_tools\n")
 
         # Copy existing "third_party/python/" vendored files
         existing_vendored = os.path.join(topsrcdir, "third_party", "python")
         work_vendored = os.path.join(work_dir, "third_party", "python")
         shutil.copytree(existing_vendored, work_vendored)
 
         # Copy "mach" module so that `VirtualenvManager` can populate itself.
         # This is needed because "topsrcdir" is used in this test both for determining
--- a/python/mozbuild/mozbuild/virtualenv.py
+++ b/python/mozbuild/mozbuild/virtualenv.py
@@ -188,17 +188,19 @@ class VirtualenvManager(VirtualenvHelper
         # * If the metadata file doesn't exist, then the virtualenv wasn't fully
         #   built
         # * If the "hex_version" doesn't match, then the system Python has changed/been
         #   upgraded.
         existing_metadata = MozVirtualenvMetadata.from_path(self._metadata.file_path)
         if existing_metadata != self._metadata:
             return False
 
-        if env_requirements.pth_requirements and self.populate_local_paths:
+        if (
+            env_requirements.pth_requirements or env_requirements.vendored_requirements
+        ) and self.populate_local_paths:
             try:
                 with open(
                     os.path.join(self._site_packages_dir(), PTH_FILENAME)
                 ) as file:
                     pth_lines = file.read().strip().split("\n")
             except FileNotFoundError:
                 return False
 
@@ -209,16 +211,17 @@ class VirtualenvManager(VirtualenvHelper
                 for path in pth_lines
             ]
 
             required_paths = [
                 os.path.normcase(
                     os.path.abspath(os.path.join(self.topsrcdir, pth.path))
                 )
                 for pth in env_requirements.pth_requirements
+                + env_requirements.vendored_requirements
             ]
 
             if current_paths != required_paths:
                 return False
 
         if (
             env_requirements.pypi_requirements
             or env_requirements.pypi_optional_requirements
@@ -350,17 +353,20 @@ class VirtualenvManager(VirtualenvHelper
 
                 old_env_variables[k] = os.environ[k]
                 del os.environ[k]
 
             env_requirements = self._requirements()
             if self.populate_local_paths:
                 python_lib = distutils.sysconfig.get_python_lib()
                 with open(os.path.join(python_lib, PTH_FILENAME), "a") as f:
-                    for pth_requirement in env_requirements.pth_requirements:
+                    for pth_requirement in (
+                        env_requirements.pth_requirements
+                        + env_requirements.vendored_requirements
+                    ):
                         path = os.path.join(self.topsrcdir, pth_requirement.path)
                         # This path is relative to the .pth file.  Using a
                         # relative path allows the srcdir/objdir combination
                         # to be moved around (as long as the paths relative to
                         # each other remain the same).
                         f.write("{}\n".format(os.path.relpath(path, python_lib)))
 
             for pypi_requirement in env_requirements.pypi_requirements: