Bug 890097 - Part 2: InstallManifest class for managing file installs; r=glandium
authorGregory Szorc <gps@mozilla.com>
Tue, 23 Jul 2013 14:36:40 -0700
changeset 152001 d27457347dd6249b694432b6698a3f07b31194bb
parent 152000 0c91f66956b17455de9329ab8309a070bf45ef38
child 152002 a0fa8c9992a5651ea22610cf9db928ba2a4d1bab
push id2859
push userakeybl@mozilla.com
push dateMon, 16 Sep 2013 19:14:59 +0000
treeherdermozilla-beta@87d3c51cd2bf [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs890097
milestone25.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 890097 - Part 2: InstallManifest class for managing file installs; r=glandium
python/mozbuild/mozpack/files.py
python/mozbuild/mozpack/manifests.py
python/mozbuild/mozpack/test/test_files.py
python/mozbuild/mozpack/test/test_manifests.py
--- a/python/mozbuild/mozpack/files.py
+++ b/python/mozbuild/mozpack/files.py
@@ -12,17 +12,20 @@ from mozpack.executables import (
     is_executable,
     may_strip,
     strip,
     may_elfhack,
     elfhack,
 )
 from mozpack.chrome.manifest import ManifestEntry
 from io import BytesIO
-from mozpack.errors import ErrorMessage
+from mozpack.errors import (
+    ErrorMessage,
+    errors,
+)
 from mozpack.mozjar import JarReader
 import mozpack.path
 from collections import OrderedDict
 
 
 class Dest(object):
     '''
     Helper interface for BaseFile.copy. The interface works as follows:
@@ -251,16 +254,38 @@ class AbsoluteSymlinkFile(File):
         except EnvironmentError:
             os.remove(temp_dest)
             raise
 
         os.rename(temp_dest, dest)
         return True
 
 
+class RequiredExistingFile(BaseFile):
+    '''
+    File class that represents a file that must exist in the destination.
+
+    The purpose of this class is to account for files that are installed
+    via external means.
+
+    When asked to copy, this class does nothing because nothing is known about
+    the source file/data. However, since this file is required, we do validate
+    that the destination path exists.
+    '''
+    def copy(self, dest, skip_if_older=True):
+        if isinstance(dest, basestring):
+            dest = Dest(dest)
+        else:
+            assert isinstance(dest, Dest)
+
+        if not dest.exists():
+            errors.fatal("Required existing file doesn't exist: %s" %
+                dest.path)
+
+
 class GeneratedFile(BaseFile):
     '''
     File class for content with no previous existence on the filesystem.
     '''
     def __init__(self, content):
         self.content = content
 
     def open(self):
--- a/python/mozbuild/mozpack/manifests.py
+++ b/python/mozbuild/mozpack/manifests.py
@@ -2,16 +2,21 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 from contextlib import contextmanager
 
 from .copier import FilePurger
+from .files import (
+    AbsoluteSymlinkFile,
+    File,
+    RequiredExistingFile,
+)
 import mozpack.path as mozpath
 
 
 # This probably belongs in a more generic module. Where?
 @contextmanager
 def _auto_fileobj(path, fileobj, mode='r'):
     if path and fileobj:
         raise AssertionError('Only 1 of path or fileobj may be defined.')
@@ -96,8 +101,182 @@ class PurgeManifest(object):
         p = FilePurger()
         for entry in self.entries:
             if prepend_relpath:
                 entry = mozpath.join(self.relpath, entry)
 
             p.add(entry)
 
         return p
+
+
+class UnreadableInstallManifest(Exception):
+    """Raised when an invalid install manifest is parsed."""
+
+
+class InstallManifest(object):
+    """Describes actions to be used with a copier.FileCopier instance.
+
+    This class facilitates serialization and deserialization of data used to
+    construct a copier.FileCopier and to perform copy operations.
+
+    The manifest defines source paths, destination paths, and a mechanism by
+    which the destination file should come into existence.
+
+    Entries in the manifest correspond to the following types:
+
+      copy -- The file specified as the source path will be copied to the
+          destination path.
+
+      symlink -- The destination path will be a symlink to the source path.
+          If symlinks are not supported, a copy will be performed.
+
+      exists -- The destination path is accounted for and won't be deleted by
+          the FileCopier.
+    """
+    FIELD_SEPARATOR = '\x1f'
+
+    SYMLINK = 1
+    COPY = 2
+    REQUIRED_EXISTS = 3
+
+    def __init__(self, path=None, fileobj=None):
+        """Create a new InstallManifest entry.
+
+        If path is defined, the manifest will be populated with data from the
+        file path.
+
+        If fh is defined, the manifest will be populated with data read
+        from the specified file object.
+
+        Both path and fileobj cannot be defined.
+        """
+        self._dests = {}
+
+        if not path and not fileobj:
+            return
+
+        with _auto_fileobj(path, fileobj, 'rb') as fh:
+            self._load_from_fileobj(fh)
+
+    def _load_from_fileobj(self, fileobj):
+        version = fileobj.readline().rstrip()
+        if version != '1':
+            raise UnreadableInstallManifest('Unknown manifest version: ' %
+                version)
+
+        for line in fileobj:
+            line = line.rstrip()
+
+            fields = line.split(self.FIELD_SEPARATOR)
+
+            record_type = int(fields[0])
+
+            if record_type == self.SYMLINK:
+                dest, source= fields[1:]
+                self.add_symlink(source, dest)
+                continue
+
+            if record_type == self.COPY:
+                dest, source = fields[1:]
+                self.add_copy(source, dest)
+                continue
+
+            if record_type == self.REQUIRED_EXISTS:
+                _, path = fields
+                self.add_required_exists(path)
+                continue
+
+            raise UnreadableInstallManifest('Unknown record type: %d' %
+                record_type)
+
+    def __len__(self):
+        return len(self._dests)
+
+    def __contains__(self, item):
+        return item in self._dests
+
+    def __eq__(self, other):
+        return isinstance(other, InstallManifest) and self._dests == other._dests
+
+    def __neq__(self, other):
+        return not self.__eq__(other)
+
+    def __ior__(self, other):
+        if not isinstance(other, InstallManifest):
+            raise ValueError('Can only | with another instance of InstallManifest.')
+
+        for dest in sorted(other._dests):
+            self._add_entry(dest, other._dests[dest])
+
+        return self
+
+    def write(self, path=None, fileobj=None):
+        """Serialize this manifest to a file or file object.
+
+        If path is specified, that file will be written to. If fileobj is specified,
+        the serialized content will be written to that file object.
+
+        It is an error if both are specified.
+        """
+        with _auto_fileobj(path, fileobj, 'wb') as fh:
+            fh.write('1\n')
+
+            for dest in sorted(self._dests):
+                entry = self._dests[dest]
+
+                parts = ['%d' % entry[0], dest]
+                parts.extend(entry[1:])
+                fh.write('%s\n' % self.FIELD_SEPARATOR.join(
+                    p.encode('utf-8') for p in parts))
+
+    def add_symlink(self, source, dest):
+        """Add a symlink to this manifest.
+
+        dest will be a symlink to source.
+        """
+        self._add_entry(dest, (self.SYMLINK, source))
+
+    def add_copy(self, source, dest):
+        """Add a copy to this manifest.
+
+        source will be copied to dest.
+        """
+        self._add_entry(dest, (self.COPY, source))
+
+    def add_required_exists(self, dest):
+        """Record that a destination file may exist.
+
+        This effectively prevents the listed file from being deleted.
+        """
+        self._add_entry(dest, (self.REQUIRED_EXISTS,))
+
+    def _add_entry(self, dest, entry):
+        if dest in self._dests:
+            raise ValueError('Item already in manifest: %s' % dest)
+
+        self._dests[dest] = entry
+
+    def populate_registry(self, registry):
+        """Populate a mozpack.copier.FileRegistry instance with data from us.
+
+        The caller supplied a FileRegistry instance (or at least something that
+        conforms to its interface) and that instance is populated with data
+        from this manifest.
+        """
+        for dest in sorted(self._dests):
+            entry = self._dests[dest]
+            install_type = entry[0]
+
+            if install_type == self.SYMLINK:
+                registry.add(dest, AbsoluteSymlinkFile(entry[1]))
+                continue
+
+            if install_type == self.COPY:
+                registry.add(dest, File(entry[1]))
+                continue
+
+            if install_type == self.REQUIRED_EXISTS:
+                registry.add(dest, RequiredExistingFile())
+                continue
+
+            raise Exception('Unknown install type defined in manifest: %d' %
+                install_type)
--- a/python/mozbuild/mozpack/test/test_files.py
+++ b/python/mozbuild/mozpack/test/test_files.py
@@ -1,23 +1,25 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from mozpack.errors import ErrorMessage
 from mozpack.files import (
     AbsoluteSymlinkFile,
     Dest,
     File,
     GeneratedFile,
     DeflatedFile,
     ManifestFile,
     XPTFile,
     MinifiedProperties,
     FileFinder,
     JarFinder,
+    RequiredExistingFile,
 )
 from mozpack.mozjar import (
     JarReader,
     JarWriter,
 )
 from mozpack.chrome.manifest import (
     ManifestContent,
     ManifestResource,
@@ -318,16 +320,31 @@ class TestAbsoluteSymlinkFile(TestWithTm
 
         s = AbsoluteSymlinkFile(source)
         self.assertFalse(s.copy(dest))
 
         link = os.readlink(dest)
         self.assertEqual(link, source)
 
 
+class TestRequiredExistingFile(TestWithTmpDir):
+    def test_missing_dest(self):
+        with self.assertRaisesRegexp(ErrorMessage, 'Required existing file'):
+            f = RequiredExistingFile()
+            f.copy(self.tmppath('dest'))
+
+    def test_existing_dest(self):
+        p = self.tmppath('dest')
+        with open(p, 'a'):
+            pass
+
+        f = RequiredExistingFile()
+        f.copy(p)
+
+
 class TestGeneratedFile(TestWithTmpDir):
     def test_generated_file(self):
         '''
         Check that GeneratedFile.copy yields the proper content in the
         destination file in all situations that trigger different code paths
         (see TestFile.test_file)
         '''
         dest = self.tmppath('dest')
--- a/python/mozbuild/mozpack/test/test_manifests.py
+++ b/python/mozbuild/mozpack/test/test_manifests.py
@@ -1,20 +1,24 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 import os
-import unittest
 
 import mozunit
 
+from mozpack.copier import (
+    FileCopier,
+    FileRegistry,
+)
 from mozpack.manifests import (
+    InstallManifest,
     PurgeManifest,
     UnreadablePurgeManifest,
 )
 from mozpack.test.test_files import TestWithTmpDir
 
 
 class TestPurgeManifest(TestWithTmpDir):
     def test_construct(self):
@@ -42,10 +46,131 @@ class TestPurgeManifest(TestWithTmpDir):
         with open(p, 'wt') as fh:
             fh.write('2\n')
             fh.write('not relevant')
 
         with self.assertRaises(UnreadablePurgeManifest):
             PurgeManifest(path=p)
 
 
+class TestInstallManifest(TestWithTmpDir):
+    def test_construct(self):
+        m = InstallManifest()
+        self.assertEqual(len(m), 0)
+
+    def test_adds(self):
+        m = InstallManifest()
+        m.add_symlink('s_source', 's_dest')
+        m.add_copy('c_source', 'c_dest')
+        m.add_required_exists('e_dest')
+
+        self.assertEqual(len(m), 3)
+        self.assertIn('s_dest', m)
+        self.assertIn('c_dest', m)
+        self.assertIn('e_dest', m)
+
+        with self.assertRaises(ValueError):
+            m.add_symlink('s_other', 's_dest')
+
+        with self.assertRaises(ValueError):
+            m.add_copy('c_other', 'c_dest')
+
+        with self.assertRaises(ValueError):
+            m.add_required_exists('e_dest')
+
+    def _get_test_manifest(self):
+        m = InstallManifest()
+        m.add_symlink(self.tmppath('s_source'), 's_dest')
+        m.add_copy(self.tmppath('c_source'), 'c_dest')
+        m.add_required_exists('e_dest')
+
+        return m
+
+    def test_serialization(self):
+        m = self._get_test_manifest()
+
+        p = self.tmppath('m')
+        m.write(path=p)
+        self.assertTrue(os.path.isfile(p))
+
+        with open(p, 'rb') as fh:
+            c = fh.read()
+
+        self.assertEqual(c.count('\n'), 4)
+
+        lines = c.splitlines()
+        self.assertEqual(len(lines), 4)
+
+        self.assertEqual(lines[0], '1')
+        self.assertEqual(lines[1], '2\x1fc_dest\x1f%s' %
+            self.tmppath('c_source'))
+        self.assertEqual(lines[2], '3\x1fe_dest')
+        self.assertEqual(lines[3], '1\x1fs_dest\x1f%s' %
+            self.tmppath('s_source'))
+
+        m2 = InstallManifest(path=p)
+        self.assertEqual(m, m2)
+        p2 = self.tmppath('m2')
+        m2.write(path=p2)
+
+        with open(p2, 'rb') as fh:
+            c2 = fh.read()
+
+        self.assertEqual(c, c2)
+
+    def test_populate_registry(self):
+        m = self._get_test_manifest()
+        r = FileRegistry()
+        m.populate_registry(r)
+
+        self.assertEqual(len(r), 3)
+        self.assertEqual(r.paths(), ['c_dest', 'e_dest', 's_dest'])
+
+    def test_or(self):
+        m1 = self._get_test_manifest()
+        m2 = InstallManifest()
+        m2.add_symlink('s_source2', 's_dest2')
+        m2.add_copy('c_source2', 'c_dest2')
+
+        m1 |= m2
+
+        self.assertEqual(len(m2), 2)
+        self.assertEqual(len(m1), 5)
+
+        self.assertIn('s_dest2', m1)
+        self.assertIn('c_dest2', m1)
+
+    def test_copier_application(self):
+        dest = self.tmppath('dest')
+        os.mkdir(dest)
+
+        to_delete = self.tmppath('dest/to_delete')
+        with open(to_delete, 'a'):
+            pass
+
+        with open(self.tmppath('s_source'), 'wt') as fh:
+            fh.write('symlink!')
+
+        with open(self.tmppath('c_source'), 'wt') as fh:
+            fh.write('copy!')
+
+        with open(self.tmppath('dest/e_dest'), 'a'):
+            pass
+
+        m = self._get_test_manifest()
+        c = FileCopier()
+        m.populate_registry(c)
+        c.copy(dest)
+
+        self.assertTrue(os.path.exists(self.tmppath('dest/s_dest')))
+        self.assertTrue(os.path.exists(self.tmppath('dest/c_dest')))
+        self.assertTrue(os.path.exists(self.tmppath('dest/e_dest')))
+        self.assertFalse(os.path.exists(to_delete))
+
+        with open(self.tmppath('dest/s_dest'), 'rt') as fh:
+            self.assertEqual(fh.read(), 'symlink!')
+
+        with open(self.tmppath('dest/c_dest'), 'rt') as fh:
+            self.assertEqual(fh.read(), 'copy!')
+
+
 if __name__ == '__main__':
     mozunit.main()