author | Andrew Halberstadt <ahalberstadt@mozilla.com> |
Mon, 22 Jun 2020 13:20:57 +0000 | |
changeset 536608 | eba33e59bc8f5c2a42a8e19e7de84ab62e21f048 |
parent 536607 | 97e16bc7c6041592170c4a45252593ecb866d3d5 |
child 536609 | 89c9510978060de5aee8c1ca04bb80967e363b03 |
push id | 37531 |
push user | nbeleuzu@mozilla.com |
push date | Tue, 23 Jun 2020 03:44:39 +0000 |
treeherder | mozilla-central@b1146cce5053 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | rstewart |
bugs | 1646427 |
milestone | 79.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
python/mozbuild/mozbuild/vendor/mach_commands.py | file | annotate | diff | comparison | revisions | |
python/mozbuild/mozbuild/vendor/vendor_python.py | file | annotate | diff | comparison | revisions |
--- a/python/mozbuild/mozbuild/vendor/mach_commands.py +++ b/python/mozbuild/mozbuild/vendor/mach_commands.py @@ -144,25 +144,32 @@ Please commit or stash these changes bef vendor_command = self._spawn(VendorRust) vendor_command.vendor(**kwargs) # ===================================================================== @SubCommand( "vendor", "python", - description="Vendor Python packages from pypi.org into third_party/python", + description="Vendor Python packages from pypi.org into third_party/python. " + "Some extra files like docs and tests will automatically be excluded.", ) @CommandArgument( "--with-windows-wheel", action="store_true", help="Vendor a wheel for Windows along with the source package", default=False, ) @CommandArgument( + "--keep-extra-files", + action="store_true", + default=False, + help="Keep all files, including tests and documentation.", + ) + @CommandArgument( "packages", default=None, nargs="*", help="Packages to vendor. If omitted, packages and their dependencies " "defined in Pipfile.lock will be vendored. If Pipfile has been modified, " "then Pipfile.lock will be regenerated. Note that transient dependencies " "may be updated when running this command.", )
--- a/python/mozbuild/mozbuild/vendor/vendor_python.py +++ b/python/mozbuild/mozbuild/vendor/vendor_python.py @@ -11,17 +11,17 @@ import subprocess import mozfile import mozpack.path as mozpath from mozbuild.base import MozbuildObject from mozfile import TemporaryDirectory from mozpack.files import FileFinder class VendorPython(MozbuildObject): - def vendor(self, packages=None, with_windows_wheel=False): + def vendor(self, packages=None, with_windows_wheel=False, keep_extra_files=False): self.populate_logger() self.log_manager.enable_unstructured() vendor_dir = mozpath.join(self.topsrcdir, os.path.join("third_party", "python")) packages = packages or [] if with_windows_wheel and len(packages) != 1: raise Exception( @@ -94,17 +94,17 @@ class VendorPython(MozbuildObject): "--python-version", "27", "--abi", "none", "--disable-pip-version-check", packages[0], ] ) - self._extract(tmp, vendor_dir) + self._extract(tmp, vendor_dir, keep_extra_files) shutil.copyfile(tmpspec_absolute, spec) self.repository.add_remove_files(vendor_dir) def _update_packages(self, spec, packages): for package in packages: if not all(package.partition("==")): raise Exception( @@ -128,37 +128,46 @@ class VendorPython(MozbuildObject): requirements[name] = version, [] with open(spec, "w") as f: for name, (version, comments) in sorted(requirements.items()): if comments: f.write("{}\n".format("\n".join(comments))) f.write("{}=={}\n".format(name, version)) - def _extract(self, src, dest): + def _extract(self, src, dest, keep_extra_files=False): """extract source distribution into vendor directory""" + + ignore = () + if not keep_extra_files: + ignore = ( + '*/doc', + '*/docs', + '*/test', + '*/tests', + ) finder = FileFinder(src) for path, _ in finder.find("*"): base, ext = os.path.splitext(path) if ext == ".whl": # Wheels would extract into a directory with the name of the package, but # we want the platform signifiers, minus the version number. # Wheel filenames look like: # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag} bits = base.split("-") # Remove the version number. bits.pop(1) target = os.path.join(dest, "-".join(bits)) mozfile.remove(target) # remove existing version of vendored package os.mkdir(target) - mozfile.extract(os.path.join(finder.base, path), target) + mozfile.extract(os.path.join(finder.base, path), target, ignore=ignore) else: # packages extract into package-version directory name and we strip the version - tld = mozfile.extract(os.path.join(finder.base, path), dest)[0] + tld = mozfile.extract(os.path.join(finder.base, path), dest, ignore=ignore)[0] target = os.path.join(dest, tld.rpartition("-")[0]) mozfile.remove(target) # remove existing version of vendored package mozfile.move(tld, target) # If any files inside the vendored package were symlinks, turn them into normal files # because hg.mozilla.org forbids symlinks in the repository. link_finder = FileFinder(target) for _, f in link_finder.find("**"): if os.path.islink(f.path):