Bug 1646427 - [vendor] Exclude test and documentation dirs when vendoring Python packages, r=rstewart
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 22 Jun 2020 13:20:57 +0000
changeset 536608 eba33e59bc8f5c2a42a8e19e7de84ab62e21f048
parent 536607 97e16bc7c6041592170c4a45252593ecb866d3d5
child 536609 89c9510978060de5aee8c1ca04bb80967e363b03
push id37531
push usernbeleuzu@mozilla.com
push dateTue, 23 Jun 2020 03:44:39 +0000
treeherdermozilla-central@b1146cce5053 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1646427 - [vendor] Exclude test and documentation dirs when vendoring Python packages, r=rstewart This adds a way to exclude certain file patterns when extracting Python packages in |mach vendor python|. For now I've excluded likely test and doc dirs, though we may want to expand on the list in the future. Differential Revision: https://phabricator.services.mozilla.com/D80209
--- a/python/mozbuild/mozbuild/vendor/mach_commands.py
+++ b/python/mozbuild/mozbuild/vendor/mach_commands.py
@@ -144,25 +144,32 @@ Please commit or stash these changes bef
         vendor_command = self._spawn(VendorRust)
     # =====================================================================
-        description="Vendor Python packages from pypi.org into third_party/python",
+        description="Vendor Python packages from pypi.org into third_party/python. "
+                    "Some extra files like docs and tests will automatically be excluded.",
         help="Vendor a wheel for Windows along with the source package",
+        "--keep-extra-files",
+        action="store_true",
+        default=False,
+        help="Keep all files, including tests and documentation.",
+    )
+    @CommandArgument(
         help="Packages to vendor. If omitted, packages and their dependencies "
         "defined in Pipfile.lock will be vendored. If Pipfile has been modified, "
         "then Pipfile.lock will be regenerated. Note that transient dependencies "
         "may be updated when running this command.",
--- a/python/mozbuild/mozbuild/vendor/vendor_python.py
+++ b/python/mozbuild/mozbuild/vendor/vendor_python.py
@@ -11,17 +11,17 @@ import subprocess
 import mozfile
 import mozpack.path as mozpath
 from mozbuild.base import MozbuildObject
 from mozfile import TemporaryDirectory
 from mozpack.files import FileFinder
 class VendorPython(MozbuildObject):
-    def vendor(self, packages=None, with_windows_wheel=False):
+    def vendor(self, packages=None, with_windows_wheel=False, keep_extra_files=False):
         vendor_dir = mozpath.join(self.topsrcdir, os.path.join("third_party", "python"))
         packages = packages or []
         if with_windows_wheel and len(packages) != 1:
             raise Exception(
@@ -94,17 +94,17 @@ class VendorPython(MozbuildObject):
-                self._extract(tmp, vendor_dir)
+                self._extract(tmp, vendor_dir, keep_extra_files)
             shutil.copyfile(tmpspec_absolute, spec)
     def _update_packages(self, spec, packages):
         for package in packages:
             if not all(package.partition("==")):
                 raise Exception(
@@ -128,37 +128,46 @@ class VendorPython(MozbuildObject):
             requirements[name] = version, []
         with open(spec, "w") as f:
             for name, (version, comments) in sorted(requirements.items()):
                 if comments:
                 f.write("{}=={}\n".format(name, version))
-    def _extract(self, src, dest):
+    def _extract(self, src, dest, keep_extra_files=False):
         """extract source distribution into vendor directory"""
+        ignore = ()
+        if not keep_extra_files:
+            ignore = (
+                '*/doc',
+                '*/docs',
+                '*/test',
+                '*/tests',
+            )
         finder = FileFinder(src)
         for path, _ in finder.find("*"):
             base, ext = os.path.splitext(path)
             if ext == ".whl":
                 # Wheels would extract into a directory with the name of the package, but
                 # we want the platform signifiers, minus the version number.
                 # Wheel filenames look like:
                 # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}
                 bits = base.split("-")
                 # Remove the version number.
                 target = os.path.join(dest, "-".join(bits))
                 mozfile.remove(target)  # remove existing version of vendored package
-                mozfile.extract(os.path.join(finder.base, path), target)
+                mozfile.extract(os.path.join(finder.base, path), target, ignore=ignore)
                 # packages extract into package-version directory name and we strip the version
-                tld = mozfile.extract(os.path.join(finder.base, path), dest)[0]
+                tld = mozfile.extract(os.path.join(finder.base, path), dest, ignore=ignore)[0]
                 target = os.path.join(dest, tld.rpartition("-")[0])
                 mozfile.remove(target)  # remove existing version of vendored package
                 mozfile.move(tld, target)
             # If any files inside the vendored package were symlinks, turn them into normal files
             # because hg.mozilla.org forbids symlinks in the repository.
             link_finder = FileFinder(target)
             for _, f in link_finder.find("**"):
                 if os.path.islink(f.path):