Bug 1646427 - [vendor] Exclude test and documentation dirs when vendoring Python packages, r=rstewart
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 22 Jun 2020 13:20:57 +0000
changeset 536625 eba33e59bc8f5c2a42a8e19e7de84ab62e21f048
parent 536624 97e16bc7c6041592170c4a45252593ecb866d3d5
child 536626 89c9510978060de5aee8c1ca04bb80967e363b03
push id119582
push userahalberstadt@mozilla.com
push dateMon, 22 Jun 2020 18:42:21 +0000
treeherderautoland@eba33e59bc8f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrstewart
bugs1646427
milestone79.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1646427 - [vendor] Exclude test and documentation dirs when vendoring Python packages, r=rstewart This adds a way to exclude certain file patterns when extracting Python packages in |mach vendor python|. For now I've excluded likely test and doc dirs, though we may want to expand on the list in the future. Differential Revision: https://phabricator.services.mozilla.com/D80209
python/mozbuild/mozbuild/vendor/mach_commands.py
python/mozbuild/mozbuild/vendor/vendor_python.py
--- a/python/mozbuild/mozbuild/vendor/mach_commands.py
+++ b/python/mozbuild/mozbuild/vendor/mach_commands.py
@@ -144,25 +144,32 @@ Please commit or stash these changes bef
         vendor_command = self._spawn(VendorRust)
         vendor_command.vendor(**kwargs)
 
     # =====================================================================
 
     @SubCommand(
         "vendor",
         "python",
-        description="Vendor Python packages from pypi.org into third_party/python",
+        description="Vendor Python packages from pypi.org into third_party/python. "
+                    "Some extra files like docs and tests will automatically be excluded.",
     )
     @CommandArgument(
         "--with-windows-wheel",
         action="store_true",
         help="Vendor a wheel for Windows along with the source package",
         default=False,
     )
     @CommandArgument(
+        "--keep-extra-files",
+        action="store_true",
+        default=False,
+        help="Keep all files, including tests and documentation.",
+    )
+    @CommandArgument(
         "packages",
         default=None,
         nargs="*",
         help="Packages to vendor. If omitted, packages and their dependencies "
         "defined in Pipfile.lock will be vendored. If Pipfile has been modified, "
         "then Pipfile.lock will be regenerated. Note that transient dependencies "
         "may be updated when running this command.",
     )
--- a/python/mozbuild/mozbuild/vendor/vendor_python.py
+++ b/python/mozbuild/mozbuild/vendor/vendor_python.py
@@ -11,17 +11,17 @@ import subprocess
 import mozfile
 import mozpack.path as mozpath
 from mozbuild.base import MozbuildObject
 from mozfile import TemporaryDirectory
 from mozpack.files import FileFinder
 
 
 class VendorPython(MozbuildObject):
-    def vendor(self, packages=None, with_windows_wheel=False):
+    def vendor(self, packages=None, with_windows_wheel=False, keep_extra_files=False):
         self.populate_logger()
         self.log_manager.enable_unstructured()
 
         vendor_dir = mozpath.join(self.topsrcdir, os.path.join("third_party", "python"))
 
         packages = packages or []
         if with_windows_wheel and len(packages) != 1:
             raise Exception(
@@ -94,17 +94,17 @@ class VendorPython(MozbuildObject):
                             "--python-version",
                             "27",
                             "--abi",
                             "none",
                             "--disable-pip-version-check",
                             packages[0],
                         ]
                     )
-                self._extract(tmp, vendor_dir)
+                self._extract(tmp, vendor_dir, keep_extra_files)
 
             shutil.copyfile(tmpspec_absolute, spec)
             self.repository.add_remove_files(vendor_dir)
 
     def _update_packages(self, spec, packages):
         for package in packages:
             if not all(package.partition("==")):
                 raise Exception(
@@ -128,37 +128,46 @@ class VendorPython(MozbuildObject):
             requirements[name] = version, []
 
         with open(spec, "w") as f:
             for name, (version, comments) in sorted(requirements.items()):
                 if comments:
                     f.write("{}\n".format("\n".join(comments)))
                 f.write("{}=={}\n".format(name, version))
 
-    def _extract(self, src, dest):
+    def _extract(self, src, dest, keep_extra_files=False):
         """extract source distribution into vendor directory"""
+
+        ignore = ()
+        if not keep_extra_files:
+            ignore = (
+                '*/doc',
+                '*/docs',
+                '*/test',
+                '*/tests',
+            )
         finder = FileFinder(src)
         for path, _ in finder.find("*"):
             base, ext = os.path.splitext(path)
             if ext == ".whl":
                 # Wheels would extract into a directory with the name of the package, but
                 # we want the platform signifiers, minus the version number.
                 # Wheel filenames look like:
                 # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}
                 bits = base.split("-")
 
                 # Remove the version number.
                 bits.pop(1)
                 target = os.path.join(dest, "-".join(bits))
                 mozfile.remove(target)  # remove existing version of vendored package
                 os.mkdir(target)
-                mozfile.extract(os.path.join(finder.base, path), target)
+                mozfile.extract(os.path.join(finder.base, path), target, ignore=ignore)
             else:
                 # packages extract into package-version directory name and we strip the version
-                tld = mozfile.extract(os.path.join(finder.base, path), dest)[0]
+                tld = mozfile.extract(os.path.join(finder.base, path), dest, ignore=ignore)[0]
                 target = os.path.join(dest, tld.rpartition("-")[0])
                 mozfile.remove(target)  # remove existing version of vendored package
                 mozfile.move(tld, target)
             # If any files inside the vendored package were symlinks, turn them into normal files
             # because hg.mozilla.org forbids symlinks in the repository.
             link_finder = FileFinder(target)
             for _, f in link_finder.find("**"):
                 if os.path.islink(f.path):