author | Nathan Froyd <froydnj@mozilla.com> |
Tue, 14 Feb 2017 16:12:19 -0500 | |
changeset 342904 | 8b9b3653e416f3de771c824c5ee5290d8961ff04 |
parent 342903 | 08891e1a5977b6a3eb5d0ec8e659c036395e2e36 |
child 342905 | 56318bc8af9b7d9a64644d9b6e780fdb8dc5ed6b |
push id | 31366 |
push user | cbook@mozilla.com |
push date | Wed, 15 Feb 2017 11:25:19 +0000 |
treeherder | mozilla-central@c0807d6938c1 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | ted.mielczarek |
bugs | 1306078 |
milestone | 54.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
python/mozbuild/mozbuild/mach_commands.py | file | annotate | diff | comparison | revisions | |
python/mozbuild/mozbuild/vendor_rust.py | file | annotate | diff | comparison | revisions |
--- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -1589,12 +1589,15 @@ class Vendor(MachCommandBase): self.parser.print_usage() sys.exit(1) @SubCommand('vendor', 'rust', description='Vendor rust crates from crates.io into third_party/rust') @CommandArgument('--ignore-modified', action='store_true', help='Ignore modified files in current checkout', default=False) + @CommandArgument('--build-peers-said-large-imports-were-ok', action='store_true', + help='Permit overly-large files to be added to the repository', + default=False) def vendor_rust(self, **kwargs): from mozbuild.vendor_rust import VendorRust vendor_command = self._spawn(VendorRust) vendor_command.vendor(**kwargs)
--- a/python/mozbuild/mozbuild/vendor_rust.py +++ b/python/mozbuild/mozbuild/vendor_rust.py @@ -75,17 +75,18 @@ Please commit or stash these changes bef return { 'OPENSSL_INCLUDE_DIR': '/usr/local/opt/openssl/include', 'DEP_OPENSSL_INCLUDE': '/usr/local/opt/openssl/include', } self.log(logging.ERROR, 'openssl', {}, "OpenSSL not found!") return None - def vendor(self, ignore_modified=False): + def vendor(self, ignore_modified=False, + build_peers_said_large_imports_were_ok=False): self.populate_logger() self.log_manager.enable_unstructured() if not ignore_modified: self.check_modified_files() cargo = self.get_cargo_path() if not self.check_cargo_version(cargo): self.log(logging.ERROR, 'cargo_version', {}, 'Cargo >= 0.13 required (install Rust 1.12 or newer)') return @@ -94,26 +95,62 @@ Please commit or stash these changes bef have_vendor = any(l.strip() == 'vendor' for l in subprocess.check_output([cargo, '--list']).splitlines()) if not have_vendor: self.log(logging.INFO, 'installing', {}, 'Installing cargo-vendor') env = self.check_openssl() self.run_process(args=[cargo, 'install', 'cargo-vendor'], append_env=env) else: self.log(logging.DEBUG, 'cargo_vendor', {}, 'cargo-vendor already intalled') - vendor_dir = mozpath.join(self.topsrcdir, 'third_party/rust') + relative_vendor_dir = 'third_party/rust' + vendor_dir = mozpath.join(self.topsrcdir, relative_vendor_dir) self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % vendor_dir) mozfile.remove(vendor_dir) # Once we require a new enough cargo to switch to workspaces, we can # just do this once on the workspace root crate. crates_and_roots = ( ('gkrust', 'toolkit/library/rust'), ('gkrust-gtest', 'toolkit/library/gtest/rust'), ('mozjs_sys', 'js/src'), ) for (lib, crate_root) in crates_and_roots: path = mozpath.join(self.topsrcdir, crate_root) # We do an |update -p| here to regenerate the Cargo.lock file with minimal changes. See bug 1324462 self._run_command_in_srcdir(args=[cargo, 'update', '--manifest-path', mozpath.join(path, 'Cargo.toml'), '-p', lib]) self._run_command_in_srcdir(args=[cargo, 'vendor', '--sync', mozpath.join(path, 'Cargo.lock'), vendor_dir]) - #TODO: print stats on size of files added/removed, warn or error - # when adding very large files (bug 1306078) self.repository.add_remove_files(vendor_dir) + + large_files = set() + cumulative_added_size = 0 + for f in self.repository.get_added_files(): + path = mozpath.join(self.topsrcdir, f) + size = os.stat(path).st_size + cumulative_added_size += size + if size > FILESIZE_LIMIT: + large_files.add(f) + + # Forcefully complain about large files being added, as history has + # shown that large-ish files typically are not needed. + if large_files and not build_peers_said_large_imports_were_ok: + self.log(logging.ERROR, 'filesize_check', {}, + '''The following files exceed the filesize limit of {size}: + +{files} + +Please find a way to reduce the sizes of these files or talk to a build +peer about the particular large files you are adding. + +The changes from `mach vendor rust` will NOT be added to version control. +'''.format(files='\n'.join(sorted(large_files)), size=FILESIZE_LIMIT)) + self.repository.forget_add_remove_files() + sys.exit(1) + + # Only warn for large imports, since we may just have large code + # drops from time to time (e.g. importing features into m-c). + SIZE_WARN_THRESHOLD = 5 * 1024 * 1024 + if cumulative_added_size >= SIZE_WARN_THRESHOLD: + self.log(logging.WARN, 'filesize_check', {}, + '''Your changes add {size} bytes of added files. + +Please consider finding ways to reduce the size of the vendored packages. +For instance, check the vendored packages for unusually large test or +benchmark files that don't need to be published to crates.io and submit +a pull request upstream to ignore those files when publishing.'''.format(size=cumulative_added_size))