Bug 1306078 - part 3 - make `mach vendor rust` check for overly-large files; r=ted.mielczarek
authorNathan Froyd <froydnj@mozilla.com>
Tue, 14 Feb 2017 16:12:19 -0500
changeset 342904 8b9b3653e416f3de771c824c5ee5290d8961ff04
parent 342903 08891e1a5977b6a3eb5d0ec8e659c036395e2e36
child 342905 56318bc8af9b7d9a64644d9b6e780fdb8dc5ed6b
push id31366
push usercbook@mozilla.com
push dateWed, 15 Feb 2017 11:25:19 +0000
treeherdermozilla-central@c0807d6938c1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1306078 - part 3 - make `mach vendor rust` check for overly-large files; r=ted.mielczarek When vendoring third-party files, we'd like an explicit notice/review when said files contain a "large file". This commit adds such checks for files vendored via `mach vendor rust`. As we don't yet have a server-side hook in place to prevent large files from being added, we just have a command-line flag that people are expected to use, on the honor system, to permit large files to be added when vendoring.
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1589,12 +1589,15 @@ class Vendor(MachCommandBase):
     @SubCommand('vendor', 'rust',
                 description='Vendor rust crates from crates.io into third_party/rust')
     @CommandArgument('--ignore-modified', action='store_true',
         help='Ignore modified files in current checkout',
+    @CommandArgument('--build-peers-said-large-imports-were-ok', action='store_true',
+        help='Permit overly-large files to be added to the repository',
+        default=False)
     def vendor_rust(self, **kwargs):
         from mozbuild.vendor_rust import VendorRust
         vendor_command = self._spawn(VendorRust)
--- a/python/mozbuild/mozbuild/vendor_rust.py
+++ b/python/mozbuild/mozbuild/vendor_rust.py
@@ -75,17 +75,18 @@ Please commit or stash these changes bef
             return {
                  'OPENSSL_INCLUDE_DIR': '/usr/local/opt/openssl/include',
                  'DEP_OPENSSL_INCLUDE': '/usr/local/opt/openssl/include',
         self.log(logging.ERROR, 'openssl', {}, "OpenSSL not found!")
         return None
-    def vendor(self, ignore_modified=False):
+    def vendor(self, ignore_modified=False,
+               build_peers_said_large_imports_were_ok=False):
         if not ignore_modified:
         cargo = self.get_cargo_path()
         if not self.check_cargo_version(cargo):
             self.log(logging.ERROR, 'cargo_version', {}, 'Cargo >= 0.13 required (install Rust 1.12 or newer)')
@@ -94,26 +95,62 @@ Please commit or stash these changes bef
         have_vendor = any(l.strip() == 'vendor' for l in subprocess.check_output([cargo, '--list']).splitlines())
         if not have_vendor:
             self.log(logging.INFO, 'installing', {}, 'Installing cargo-vendor')
             env = self.check_openssl()
             self.run_process(args=[cargo, 'install', 'cargo-vendor'],
             self.log(logging.DEBUG, 'cargo_vendor', {}, 'cargo-vendor already intalled')
-        vendor_dir = mozpath.join(self.topsrcdir, 'third_party/rust')
+        relative_vendor_dir = 'third_party/rust'
+        vendor_dir = mozpath.join(self.topsrcdir, relative_vendor_dir)
         self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % vendor_dir)
         # Once we require a new enough cargo to switch to workspaces, we can
         # just do this once on the workspace root crate.
         crates_and_roots = (
             ('gkrust', 'toolkit/library/rust'),
             ('gkrust-gtest', 'toolkit/library/gtest/rust'),
             ('mozjs_sys', 'js/src'),
         for (lib, crate_root) in crates_and_roots:
             path = mozpath.join(self.topsrcdir, crate_root)
             # We do an |update -p| here to regenerate the Cargo.lock file with minimal changes. See bug 1324462
             self._run_command_in_srcdir(args=[cargo, 'update', '--manifest-path', mozpath.join(path, 'Cargo.toml'), '-p', lib])
             self._run_command_in_srcdir(args=[cargo, 'vendor', '--sync', mozpath.join(path, 'Cargo.lock'), vendor_dir])
-        #TODO: print stats on size of files added/removed, warn or error
-        # when adding very large files (bug 1306078)
+        large_files = set()
+        cumulative_added_size = 0
+        for f in self.repository.get_added_files():
+            path = mozpath.join(self.topsrcdir, f)
+            size = os.stat(path).st_size
+            cumulative_added_size += size
+            if size > FILESIZE_LIMIT:
+                large_files.add(f)
+        # Forcefully complain about large files being added, as history has
+        # shown that large-ish files typically are not needed.
+        if large_files and not build_peers_said_large_imports_were_ok:
+            self.log(logging.ERROR, 'filesize_check', {},
+                     '''The following files exceed the filesize limit of {size}:
+Please find a way to reduce the sizes of these files or talk to a build
+peer about the particular large files you are adding.
+The changes from `mach vendor rust` will NOT be added to version control.
+'''.format(files='\n'.join(sorted(large_files)), size=FILESIZE_LIMIT))
+            self.repository.forget_add_remove_files()
+            sys.exit(1)
+        # Only warn for large imports, since we may just have large code
+        # drops from time to time (e.g. importing features into m-c).
+        SIZE_WARN_THRESHOLD = 5 * 1024 * 1024
+        if cumulative_added_size >= SIZE_WARN_THRESHOLD:
+            self.log(logging.WARN, 'filesize_check', {},
+                     '''Your changes add {size} bytes of added files.
+Please consider finding ways to reduce the size of the vendored packages.
+For instance, check the vendored packages for unusually large test or
+benchmark files that don't need to be published to crates.io and submit
+a pull request upstream to ignore those files when publishing.'''.format(size=cumulative_added_size))