Bug 1367852 - Defer file read to reduce peak memory use in mozjar when dumping symbols. draft
authorChris Manchester <cmanchester@mozilla.com>
Tue, 30 May 2017 12:11:21 -0700
changeset 586561 69f88ad323613e8037964ffa41547cefe37754e0
parent 585145 1bfa4578aa56f768626ba278a6929e23fc48db54
child 586562 dd26eaf46e21bd663c3c03a294627d9d9d7cc83d
push id61460
push usercmanchester@mozilla.com
push dateTue, 30 May 2017 19:11:51 +0000
Bug 1367852 - Defer file read to reduce peak memory use in mozjar when dumping symbols. Passing a file object intead of a string of the file's contents defers reading the file to a place Python can free the resulting string earlier, reducing peak memory consumption when packaging symbols and avoiding a MemoryError on Windows 32 builders in automation. MozReview-Commit-ID: H0R6BbjwhOu
--- a/python/mozbuild/mozbuild/action/symbols_archive.py
+++ b/python/mozbuild/mozbuild/action/symbols_archive.py
@@ -20,17 +20,17 @@ def make_archive(archive_name, base, exc
         compress = ['**/*.sym']
     archive_basename = os.path.basename(archive_name)
     with open(archive_name, 'wb') as fh:
         with JarWriter(fileobj=fh, optimize=False, compress_level=5) as writer:
             for pat in include:
                 for p, f in finder.find(pat):
                     print('  Adding to "%s":\n\t"%s"' % (archive_basename, p))
                     should_compress = any(mozpath.match(p, pat) for pat in compress)
-                    writer.add(p.encode('utf-8'), f.read(), mode=f.mode,
+                    writer.add(p.encode('utf-8'), f, mode=f.mode,
                                compress=should_compress, skip_duplicates=True)
 def main(argv):
     parser = argparse.ArgumentParser(description='Produce a symbols archive')
     parser.add_argument('archive', help='Which archive to generate')
     parser.add_argument('base', help='Base directory to package')
     parser.add_argument('--exclude', default=[], action='append', help='File patterns to exclude')
     parser.add_argument('--include', default=[], action='append', help='File patterns to include')