Bug 1359965 - Support mozpack.file.BaseFile in create_tar_from_files; r=glandium
authorGregory Szorc <gps@mozilla.com>
Mon, 08 May 2017 17:00:20 -0700
changeset 412394 0a94b9164a649af95e7e645669bf6b2a1883745c
parent 412393 291fe1a6a375d8f2a95908f3a28f93bfe16ec4f3
child 412395 11ba6213be5a745d70e5c6fdcd036bafb1ac2718
push id1490
push usermtabara@mozilla.com
push dateMon, 31 Jul 2017 14:08:16 +0000
treeherdermozilla-release@70e32e6bf15e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1359965
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1359965 - Support mozpack.file.BaseFile in create_tar_from_files; r=glandium This allows us to write files coming from a finder or other source that isn't directly the filesystem. MozReview-Commit-ID: KhPSD0JYzsQ
python/mozbuild/mozpack/archive.py
python/mozbuild/mozpack/files.py
python/mozbuild/mozpack/test/test_archive.py
--- a/python/mozbuild/mozpack/archive.py
+++ b/python/mozbuild/mozpack/archive.py
@@ -1,64 +1,78 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import
 
 import bz2
 import gzip
+import io
 import stat
 import tarfile
 
+from .files import (
+    BaseFile,
+)
 
 # 2016-01-01T00:00:00+0000
 DEFAULT_MTIME = 1451606400
 
 
 def create_tar_from_files(fp, files):
     """Create a tar file deterministically.
 
     Receives a dict mapping names of files in the archive to local filesystem
-    paths.
+    paths or ``mozpack.files.BaseFile`` instances.
 
     The files will be archived and written to the passed file handle opened
     for writing.
 
     Only regular files can be written.
 
-    FUTURE accept mozpack.files classes for writing
     FUTURE accept a filename argument (or create APIs to write files)
     """
     with tarfile.open(name='', mode='w', fileobj=fp, dereference=True) as tf:
-        for archive_path, fs_path in sorted(files.items()):
-            ti = tf.gettarinfo(fs_path, archive_path)
+        for archive_path, f in sorted(files.items()):
+            if isinstance(f, BaseFile):
+                ti = tarfile.TarInfo(archive_path)
+                ti.mode = f.mode or 0644
+                ti.type = tarfile.REGTYPE
+            else:
+                ti = tf.gettarinfo(f, archive_path)
 
             if not ti.isreg():
-                raise ValueError('not a regular file: %s' % fs_path)
+                raise ValueError('not a regular file: %s' % f)
 
             # Disallow setuid and setgid bits. This is an arbitrary restriction.
             # However, since we set uid/gid to root:root, setuid and setgid
             # would be a glaring security hole if the archive were
             # uncompressed as root.
             if ti.mode & (stat.S_ISUID | stat.S_ISGID):
                 raise ValueError('cannot add file with setuid or setgid set: '
-                                 '%s' % fs_path)
+                                 '%s' % f)
 
             # Set uid, gid, username, and group as deterministic values.
             ti.uid = 0
             ti.gid = 0
             ti.uname = ''
             ti.gname = ''
 
             # Set mtime to a constant value.
             ti.mtime = DEFAULT_MTIME
 
-            with open(fs_path, 'rb') as fh:
-                tf.addfile(ti, fh)
+            if isinstance(f, BaseFile):
+                ti.size = f.size()
+                # tarfile wants to pass a size argument to read(). So just
+                # wrap/buffer in a proper file object interface.
+                tf.addfile(ti, f.open())
+            else:
+                with open(f, 'rb') as fh:
+                    tf.addfile(ti, fh)
 
 
 def create_tar_gz_from_files(fp, files, filename=None, compresslevel=9):
     """Create a tar.gz file deterministically from files.
 
     This is a glorified wrapper around ``create_tar_from_files`` that
     adds gzip compression.
 
--- a/python/mozbuild/mozpack/files.py
+++ b/python/mozbuild/mozpack/files.py
@@ -214,16 +214,24 @@ class BaseFile(object):
         a custom file-like object.
         '''
         assert self.path is not None
         return open(self.path, 'rb')
 
     def read(self):
         raise NotImplementedError('BaseFile.read() not implemented. Bug 1170329.')
 
+    def size(self):
+        """Returns size of the entry.
+
+        Derived classes are highly encouraged to override this with a more
+        optimal implementation.
+        """
+        return len(self.read())
+
     @property
     def mode(self):
         '''
         Return the file's unix mode, or None if it has no meaning.
         '''
         return None
 
 
@@ -245,16 +253,19 @@ class File(BaseFile):
         mode = os.stat(self.path).st_mode
         return self.normalize_mode(mode)
 
     def read(self):
         '''Return the contents of the file.'''
         with open(self.path, 'rb') as fh:
             return fh.read()
 
+    def size(self):
+        return os.stat(self.path).st_size
+
 
 class ExecutableFile(File):
     '''
     File class for executable and library files on OS/2, OS/X and ELF systems.
     (see mozpack.executables.is_executable documentation).
     '''
     def copy(self, dest, skip_if_older=True):
         real_dest = dest
@@ -492,16 +503,22 @@ class GeneratedFile(BaseFile):
     File class for content with no previous existence on the filesystem.
     '''
     def __init__(self, content):
         self.content = content
 
     def open(self):
         return BytesIO(self.content)
 
+    def read(self):
+        return self.content
+
+    def size(self):
+        return len(self.content)
+
 
 class DeflatedFile(BaseFile):
     '''
     File class for members of a jar archive. DeflatedFile.copy() effectively
     extracts the file from the jar archive.
     '''
     def __init__(self, file):
         from mozpack.mozjar import JarFileReader
--- a/python/mozbuild/mozpack/test/test_archive.py
+++ b/python/mozbuild/mozpack/test/test_archive.py
@@ -13,16 +13,19 @@ import tempfile
 import unittest
 
 from mozpack.archive import (
     DEFAULT_MTIME,
     create_tar_from_files,
     create_tar_gz_from_files,
     create_tar_bz2_from_files,
 )
+from mozpack.files import (
+    GeneratedFile,
+)
 
 from mozunit import main
 
 
 MODE_STANDARD = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
 
 
 def file_hash(path):
@@ -36,29 +39,32 @@ def file_hash(path):
 
     return h.hexdigest()
 
 
 class TestArchive(unittest.TestCase):
     def _create_files(self, root):
         files = {}
         for i in range(10):
-            p = os.path.join(root, b'file%d' % i)
+            p = os.path.join(root, b'file%02d' % i)
             with open(p, 'wb') as fh:
-                fh.write(b'file%d' % i)
+                fh.write(b'file%02d' % i)
             # Need to set permissions or umask may influence testing.
             os.chmod(p, MODE_STANDARD)
-            files[b'file%d' % i] = p
+            files[b'file%02d' % i] = p
+
+        for i in range(10):
+            files[b'file%02d' % (i + 10)] = GeneratedFile('file%02d' % (i + 10))
 
         return files
 
     def _verify_basic_tarfile(self, tf):
-        self.assertEqual(len(tf.getmembers()), 10)
+        self.assertEqual(len(tf.getmembers()), 20)
 
-        names = ['file%d' % i for i in range(10)]
+        names = ['file%02d' % i for i in range(20)]
         self.assertEqual(tf.getnames(), names)
 
         for ti in tf.getmembers():
             self.assertEqual(ti.uid, 0)
             self.assertEqual(ti.gid, 0)
             self.assertEqual(ti.uname, '')
             self.assertEqual(ti.gname, '')
             self.assertEqual(ti.mode, MODE_STANDARD)
@@ -101,17 +107,17 @@ class TestArchive(unittest.TestCase):
         try:
             files = self._create_files(d)
 
             tp = os.path.join(d, 'test.tar')
             with open(tp, 'wb') as fh:
                 create_tar_from_files(fh, files)
 
             # Output should be deterministic.
-            self.assertEqual(file_hash(tp), 'cd16cee6f13391abd94dfa435d2633b61ed727f1')
+            self.assertEqual(file_hash(tp), '01cd314e277f060e98c7de6c8ea57f96b3a2065c')
 
             with tarfile.open(tp, 'r') as tf:
                 self._verify_basic_tarfile(tf)
 
         finally:
             shutil.rmtree(d)
 
     def test_executable_preserved(self):
@@ -139,51 +145,51 @@ class TestArchive(unittest.TestCase):
         d = tempfile.mkdtemp()
         try:
             files = self._create_files(d)
 
             gp = os.path.join(d, 'test.tar.gz')
             with open(gp, 'wb') as fh:
                 create_tar_gz_from_files(fh, files)
 
-            self.assertEqual(file_hash(gp), 'acb602239c1aeb625da5e69336775609516d60f5')
+            self.assertEqual(file_hash(gp), '7c4da5adc5088cdf00911d5daf9a67b15de714b7')
 
             with tarfile.open(gp, 'r:gz') as tf:
                 self._verify_basic_tarfile(tf)
 
         finally:
             shutil.rmtree(d)
 
     def test_tar_gz_name(self):
         d = tempfile.mkdtemp()
         try:
             files = self._create_files(d)
 
             gp = os.path.join(d, 'test.tar.gz')
             with open(gp, 'wb') as fh:
                 create_tar_gz_from_files(fh, files, filename='foobar', compresslevel=1)
 
-            self.assertEqual(file_hash(gp), 'fd099f96480cc1100f37baa8e89a6b820dbbcbd3')
+            self.assertEqual(file_hash(gp), '1cc8b96f0262350977c2e9d61f40a1fa76f35c52')
 
             with tarfile.open(gp, 'r:gz') as tf:
                 self._verify_basic_tarfile(tf)
 
         finally:
             shutil.rmtree(d)
 
     def test_create_tar_bz2_basic(self):
         d = tempfile.mkdtemp()
         try:
             files = self._create_files(d)
 
             bp = os.path.join(d, 'test.tar.bz2')
             with open(bp, 'wb') as fh:
                 create_tar_bz2_from_files(fh, files)
 
-            self.assertEqual(file_hash(bp), '1827ad00dfe7acf857b7a1c95ce100361e3f6eea')
+            self.assertEqual(file_hash(bp), 'eb5096d2fbb71df7b3d690001a6f2e82a5aad6a7')
 
             with tarfile.open(bp, 'r:bz2') as tf:
                 self._verify_basic_tarfile(tf)
         finally:
             shutil.rmtree(d)
 
 
 if __name__ == '__main__':