Bug 935987 - Part 3: Add preprocessed file support to install manifests; r=gps
authorBrian O'Keefe <bokeefe@alum.wpi.edu>
Sat, 14 Dec 2013 09:46:48 -0500
changeset 164110 48289161bc17e4c87c061c63b5233eba07644704
parent 164109 cbb166c4a60d198c0824eb05687ce1aa32386ba6
child 164111 a61079647dc5fcd265a3b7d2f4c8b7cf6ba45af8
push id26026
push userphilringnalda@gmail.com
push dateSat, 18 Jan 2014 23:17:27 +0000
treeherdermozilla-central@61fd0f987cf2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs935987
milestone29.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 935987 - Part 3: Add preprocessed file support to install manifests; r=gps
python/mozbuild/mozpack/manifests.py
python/mozbuild/mozpack/test/test_manifests.py
--- a/python/mozbuild/mozpack/manifests.py
+++ b/python/mozbuild/mozpack/manifests.py
@@ -1,21 +1,23 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import unicode_literals
 
 from contextlib import contextmanager
+import json
 
 from .files import (
     AbsoluteSymlinkFile,
     ExistingFile,
     File,
     FileFinder,
+    PreprocessedFile,
 )
 import mozpack.path as mozpath
 
 
 # This probably belongs in a more generic module. Where?
 @contextmanager
 def _auto_fileobj(path, fileobj, mode='r'):
     if path and fileobj:
@@ -64,51 +66,60 @@ class InstallManifest(object):
           exist.
 
       patternsymlink -- Paths matched by the expression in the source path
           will be symlinked to the destination directory.
 
       patterncopy -- Similar to patternsymlink except files are copied, not
           symlinked.
 
+      preprocess -- The file specified at the source path will be run through
+          the preprocessor, and the output will be written to the destination
+          path.
+
     Version 1 of the manifest was the initial version.
     Version 2 added optional path support
     Version 3 added support for pattern entries.
+    Version 4 added preprocessed file support.
     """
+
+    CURRENT_VERSION = 4
+
     FIELD_SEPARATOR = '\x1f'
 
     SYMLINK = 1
     COPY = 2
     REQUIRED_EXISTS = 3
     OPTIONAL_EXISTS = 4
     PATTERN_SYMLINK = 5
     PATTERN_COPY = 6
+    PREPROCESS = 7
 
     def __init__(self, path=None, fileobj=None):
         """Create a new InstallManifest entry.
 
         If path is defined, the manifest will be populated with data from the
         file path.
 
-        If fh is defined, the manifest will be populated with data read
+        If fileobj is defined, the manifest will be populated with data read
         from the specified file object.
 
         Both path and fileobj cannot be defined.
         """
         self._dests = {}
+        self._source_file = None
 
-        if not path and not fileobj:
-            return
-
-        with _auto_fileobj(path, fileobj, 'rb') as fh:
-            self._load_from_fileobj(fh)
+        if path or fileobj:
+            with _auto_fileobj(path, fileobj, 'rb') as fh:
+                self._source_file = fh.name
+                self._load_from_fileobj(fh)
 
     def _load_from_fileobj(self, fileobj):
         version = fileobj.readline().rstrip()
-        if version not in ('1', '2', '3'):
+        if version not in ('1', '2', '3', '4'):
             raise UnreadableInstallManifest('Unknown manifest version: ' %
                 version)
 
         for line in fileobj:
             line = line.rstrip()
 
             fields = line.split(self.FIELD_SEPARATOR)
 
@@ -139,16 +150,22 @@ class InstallManifest(object):
                 self.add_pattern_symlink(base, pattern, dest)
                 continue
 
             if record_type == self.PATTERN_COPY:
                 _, base, pattern, dest = fields[1:]
                 self.add_pattern_copy(base, pattern, dest)
                 continue
 
+            if record_type == self.PREPROCESS:
+                dest, source, deps, marker, defines = fields[1:]
+                self.add_preprocess(source, dest, deps, marker,
+                    self._decode_field_entry(defines))
+                continue
+
             raise UnreadableInstallManifest('Unknown record type: %d' %
                 record_type)
 
     def __len__(self):
         return len(self._dests)
 
     def __contains__(self, item):
         return item in self._dests
@@ -163,26 +180,42 @@ class InstallManifest(object):
         if not isinstance(other, InstallManifest):
             raise ValueError('Can only | with another instance of InstallManifest.')
 
         for dest in sorted(other._dests):
             self._add_entry(dest, other._dests[dest])
 
         return self
 
+    def _encode_field_entry(self, data):
+        """Converts an object into a format that can be stored in the manifest file.
+
+        Complex data types, such as ``dict``, need to be converted into a text
+        representation before they can be written to a file.
+        """
+        return json.dumps(data, sort_keys=True)
+
+    def _decode_field_entry(self, data):
+        """Restores an object from a format that can be stored in the manifest file.
+
+        Complex data types, such as ``dict``, need to be converted into a text
+        representation before they can be written to a file.
+        """
+        return json.loads(data)
+
     def write(self, path=None, fileobj=None):
         """Serialize this manifest to a file or file object.
 
         If path is specified, that file will be written to. If fileobj is specified,
         the serialized content will be written to that file object.
 
         It is an error if both are specified.
         """
         with _auto_fileobj(path, fileobj, 'wb') as fh:
-            fh.write('3\n')
+            fh.write('%d\n' % self.CURRENT_VERSION)
 
             for dest in sorted(self._dests):
                 entry = self._dests[dest]
 
                 parts = ['%d' % entry[0], dest]
                 parts.extend(entry[1:])
                 fh.write('%s\n' % self.FIELD_SEPARATOR.join(
                     p.encode('utf-8') for p in parts))
@@ -235,22 +268,34 @@ class InstallManifest(object):
     def add_pattern_copy(self, base, pattern, dest):
         """Add a pattern match that results in copies.
 
         See ``add_pattern_symlink()`` for usage.
         """
         self._add_entry(mozpath.join(base, pattern, dest),
             (self.PATTERN_COPY, base, pattern, dest))
 
+    def add_preprocess(self, source, dest, deps, marker='#', defines={}):
+        """Add a preprocessed file to this manifest.
+
+        ``source`` will be passed through preprocessor.py, and the output will be
+        written to ``dest``.
+        """
+        self._add_entry(dest,
+            (self.PREPROCESS, source, deps, marker, self._encode_field_entry(defines)))
+
     def _add_entry(self, dest, entry):
         if dest in self._dests:
             raise ValueError('Item already in manifest: %s' % dest)
 
         self._dests[dest] = entry
 
+    def _get_deps(self, dest):
+        return {self._source_file} if self._source_file else set()
+
     def populate_registry(self, registry):
         """Populate a mozpack.copier.FileRegistry instance with data from us.
 
         The caller supplied a FileRegistry instance (or at least something that
         conforms to its interface) and that instance is populated with data
         from this manifest.
         """
         for dest in sorted(self._dests):
@@ -284,10 +329,19 @@ class InstallManifest(object):
                     cls = File
 
                 for path in paths:
                     source = mozpath.join(base, path)
                     registry.add(mozpath.join(dest, path), cls(source))
 
                 continue
 
+            if install_type == self.PREPROCESS:
+                registry.add(dest, PreprocessedFile(entry[1],
+                    depfile_path=entry[2],
+                    marker=entry[3],
+                    defines=self._decode_field_entry(entry[4]),
+                    extra_depends=self._get_deps(dest)))
+
+                continue
+
             raise Exception('Unknown install type defined in manifest: %d' %
                 install_type)
--- a/python/mozbuild/mozpack/test/test_manifests.py
+++ b/python/mozbuild/mozpack/test/test_manifests.py
@@ -26,45 +26,51 @@ class TestInstallManifest(TestWithTmpDir
     def test_adds(self):
         m = InstallManifest()
         m.add_symlink('s_source', 's_dest')
         m.add_copy('c_source', 'c_dest')
         m.add_required_exists('e_dest')
         m.add_optional_exists('o_dest')
         m.add_pattern_symlink('ps_base', 'ps/*', 'ps_dest')
         m.add_pattern_copy('pc_base', 'pc/**', 'pc_dest')
+        m.add_preprocess('p_source', 'p_dest', 'p_source.pp')
 
-        self.assertEqual(len(m), 6)
+        self.assertEqual(len(m), 7)
         self.assertIn('s_dest', m)
         self.assertIn('c_dest', m)
+        self.assertIn('p_dest', m)
         self.assertIn('e_dest', m)
         self.assertIn('o_dest', m)
 
         with self.assertRaises(ValueError):
             m.add_symlink('s_other', 's_dest')
 
         with self.assertRaises(ValueError):
             m.add_copy('c_other', 'c_dest')
 
         with self.assertRaises(ValueError):
+            m.add_preprocess('p_other', 'p_dest', 'p_other.pp')
+
+        with self.assertRaises(ValueError):
             m.add_required_exists('e_dest')
 
         with self.assertRaises(ValueError):
             m.add_optional_exists('o_dest')
 
         with self.assertRaises(ValueError):
             m.add_pattern_symlink('ps_base', 'ps/*', 'ps_dest')
 
         with self.assertRaises(ValueError):
             m.add_pattern_copy('pc_base', 'pc/**', 'pc_dest')
 
     def _get_test_manifest(self):
         m = InstallManifest()
         m.add_symlink(self.tmppath('s_source'), 's_dest')
         m.add_copy(self.tmppath('c_source'), 'c_dest')
+        m.add_preprocess(self.tmppath('p_source'), 'p_dest', self.tmppath('p_source.pp'), '#', {'FOO':'BAR', 'BAZ':'QUX'})
         m.add_required_exists('e_dest')
         m.add_optional_exists('o_dest')
         m.add_pattern_symlink('ps_base', '*', 'ps_dest')
         m.add_pattern_copy('pc_base', '**', 'pc_dest')
 
         return m
 
     def test_serialization(self):
@@ -72,40 +78,40 @@ class TestInstallManifest(TestWithTmpDir
 
         p = self.tmppath('m')
         m.write(path=p)
         self.assertTrue(os.path.isfile(p))
 
         with open(p, 'rb') as fh:
             c = fh.read()
 
-        self.assertEqual(c.count('\n'), 7)
+        self.assertEqual(c.count('\n'), 8)
 
         lines = c.splitlines()
-        self.assertEqual(len(lines), 7)
+        self.assertEqual(len(lines), 8)
 
-        self.assertEqual(lines[0], '3')
+        self.assertEqual(lines[0], '4')
 
         m2 = InstallManifest(path=p)
         self.assertEqual(m, m2)
         p2 = self.tmppath('m2')
         m2.write(path=p2)
 
         with open(p2, 'rb') as fh:
             c2 = fh.read()
 
         self.assertEqual(c, c2)
 
     def test_populate_registry(self):
         m = self._get_test_manifest()
         r = FileRegistry()
         m.populate_registry(r)
 
-        self.assertEqual(len(r), 4)
-        self.assertEqual(r.paths(), ['c_dest', 'e_dest', 'o_dest', 's_dest'])
+        self.assertEqual(len(r), 5)
+        self.assertEqual(r.paths(), ['c_dest', 'e_dest', 'o_dest', 'p_dest', 's_dest'])
 
     def test_pattern_expansion(self):
         source = self.tmppath('source')
         os.mkdir(source)
         os.mkdir('%s/base' % source)
         os.mkdir('%s/base/foo' % source)
 
         with open('%s/base/foo/file1' % source, 'a'):
@@ -145,43 +151,202 @@ class TestInstallManifest(TestWithTmpDir
             pass
 
         with open(self.tmppath('s_source'), 'wt') as fh:
             fh.write('symlink!')
 
         with open(self.tmppath('c_source'), 'wt') as fh:
             fh.write('copy!')
 
+        with open(self.tmppath('p_source'), 'wt') as fh:
+            fh.write('#define FOO 1\npreprocess!')
+
         with open(self.tmppath('dest/e_dest'), 'a'):
             pass
 
         with open(self.tmppath('dest/o_dest'), 'a'):
             pass
 
         m = self._get_test_manifest()
         c = FileCopier()
         m.populate_registry(c)
         result = c.copy(dest)
 
         self.assertTrue(os.path.exists(self.tmppath('dest/s_dest')))
         self.assertTrue(os.path.exists(self.tmppath('dest/c_dest')))
+        self.assertTrue(os.path.exists(self.tmppath('dest/p_dest')))
         self.assertTrue(os.path.exists(self.tmppath('dest/e_dest')))
         self.assertTrue(os.path.exists(self.tmppath('dest/o_dest')))
         self.assertFalse(os.path.exists(to_delete))
 
         with open(self.tmppath('dest/s_dest'), 'rt') as fh:
             self.assertEqual(fh.read(), 'symlink!')
 
         with open(self.tmppath('dest/c_dest'), 'rt') as fh:
             self.assertEqual(fh.read(), 'copy!')
 
+        with open(self.tmppath('dest/p_dest'), 'rt') as fh:
+            self.assertEqual(fh.read(), 'preprocess!')
+
         self.assertEqual(result.updated_files, set(self.tmppath(p) for p in (
-            'dest/s_dest', 'dest/c_dest')))
+            'dest/s_dest', 'dest/c_dest', 'dest/p_dest')))
         self.assertEqual(result.existing_files,
             set([self.tmppath('dest/e_dest'), self.tmppath('dest/o_dest')]))
         self.assertEqual(result.removed_files, {to_delete})
         self.assertEqual(result.removed_directories, set())
 
+    def test_preprocessor(self):
+        manifest = self.tmppath('m')
+        deps = self.tmppath('m.pp')
+        dest = self.tmppath('dest')
+        include = self.tmppath('p_incl')
 
+        with open(include, 'wt') as fh:
+            fh.write('#define INCL\n')
+        time = os.path.getmtime(include) - 3
+        os.utime(include, (time, time))
 
+        with open(self.tmppath('p_source'), 'wt') as fh:
+            fh.write('#ifdef FOO\n#if BAZ == QUX\nPASS1\n#endif\n#endif\n')
+            fh.write('#ifdef DEPTEST\nPASS2\n#endif\n')
+            fh.write('#include p_incl\n#ifdef INCLTEST\nPASS3\n#endif\n')
+        time = os.path.getmtime(self.tmppath('p_source')) - 3
+        os.utime(self.tmppath('p_source'), (time, time))
+
+        # Create and write a manifest with the preprocessed file, then apply it.
+        # This should write out our preprocessed file.
+        m = InstallManifest()
+        m.add_preprocess(self.tmppath('p_source'), 'p_dest', deps, '#', {'FOO':'BAR', 'BAZ':'QUX'})
+        m.write(path=manifest)
+
+        m = InstallManifest(path=manifest)
+        c = FileCopier()
+        m.populate_registry(c)
+        c.copy(dest)
+
+        self.assertTrue(os.path.exists(self.tmppath('dest/p_dest')))
+
+        with open(self.tmppath('dest/p_dest'), 'rt') as fh:
+            self.assertEqual(fh.read(), 'PASS1\n')
+
+        # Create a second manifest with the preprocessed file, then apply it.
+        # Since this manifest does not exist on the disk, there should not be a
+        # dependency on it, and the preprocessed file should not be modified.
+        m2 = InstallManifest()
+        m2.add_preprocess(self.tmppath('p_source'), 'p_dest', deps, '#', {'DEPTEST':True})
+        c = FileCopier()
+        m2.populate_registry(c)
+        result = c.copy(dest)
+
+        self.assertFalse(self.tmppath('dest/p_dest') in result.updated_files)
+        self.assertTrue(self.tmppath('dest/p_dest') in result.existing_files)
+
+        # Write out the second manifest, then load it back in from the disk.
+        # This should add the dependency on the manifest file, so our
+        # preprocessed file should be regenerated with the new defines.
+        # We also set the mtime on the destination file back, so it will be
+        # older than the manifest file.
+        m2.write(path=manifest)
+        time = os.path.getmtime(manifest) - 1
+        os.utime(self.tmppath('dest/p_dest'), (time, time))
+        m2 = InstallManifest(path=manifest)
+        c = FileCopier()
+        m2.populate_registry(c)
+        self.assertTrue(c.copy(dest))
+
+        with open(self.tmppath('dest/p_dest'), 'rt') as fh:
+            self.assertEqual(fh.read(), 'PASS2\n')
+
+        # Set the time on the manifest back, so it won't be picked up as
+        # modified in the next test
+        time = os.path.getmtime(manifest) - 1
+        os.utime(manifest, (time, time))
+
+        # Update the contents of a file included by the source file. This should
+        # cause the destination to be regenerated.
+        with open(include, 'wt') as fh:
+            fh.write('#define INCLTEST\n')
+
+        time = os.path.getmtime(include) - 1
+        os.utime(self.tmppath('dest/p_dest'), (time, time))
+        c = FileCopier()
+        m2.populate_registry(c)
+        self.assertTrue(c.copy(dest))
+
+        with open(self.tmppath('dest/p_dest'), 'rt') as fh:
+            self.assertEqual(fh.read(), 'PASS2\nPASS3\n')
+
+    def test_preprocessor_dependencies(self):
+        manifest = self.tmppath('m')
+        deps = self.tmppath('m.pp')
+        dest = self.tmppath('dest')
+        source = self.tmppath('p_source')
+        destfile = self.tmppath('dest/p_dest')
+        include = self.tmppath('p_incl')
+        os.mkdir(dest)
+
+        with open(source, 'wt') as fh:
+            fh.write('#define SRC\nSOURCE\n')
+        time = os.path.getmtime(source) - 3
+        os.utime(source, (time, time))
+
+        with open(include, 'wt') as fh:
+            fh.write('INCLUDE\n')
+        time = os.path.getmtime(source) - 3
+        os.utime(include, (time, time))
+
+        # Create and write a manifest with the preprocessed file.
+        m = InstallManifest()
+        m.add_preprocess(source, 'p_dest', deps, '#', {'FOO':'BAR', 'BAZ':'QUX'})
+        m.write(path=manifest)
+
+        time = os.path.getmtime(source) - 5
+        os.utime(manifest, (time, time))
+
+        # Now read the manifest back in, and apply it. This should write out
+        # our preprocessed file.
+        m = InstallManifest(path=manifest)
+        c = FileCopier()
+        m.populate_registry(c)
+        self.assertTrue(c.copy(dest))
+
+        with open(destfile, 'rt') as fh:
+            self.assertEqual(fh.read(), 'SOURCE\n')
+
+        # Next, modify the source to #INCLUDE another file.
+        with open(source, 'wt') as fh:
+            fh.write('SOURCE\n#include p_incl\n')
+        time = os.path.getmtime(source) - 1
+        os.utime(destfile, (time, time))
+
+        # Apply the manifest, and confirm that it also reads the newly included
+        # file.
+        m = InstallManifest(path=manifest)
+        c = FileCopier()
+        m.populate_registry(c)
+        c.copy(dest)
+
+        with open(destfile, 'rt') as fh:
+            self.assertEqual(fh.read(), 'SOURCE\nINCLUDE\n')
+
+        # Set the time on the source file back, so it won't be picked up as
+        # modified in the next test.
+        time = os.path.getmtime(source) - 1
+        os.utime(source, (time, time))
+
+        # Now, modify the include file (but not the original source).
+        with open(include, 'wt') as fh:
+            fh.write('INCLUDE MODIFIED\n')
+        time = os.path.getmtime(include) - 1
+        os.utime(destfile, (time, time))
+
+        # Apply the manifest, and confirm that the change to the include file
+        # is detected. That should cause the preprocessor to run again.
+        m = InstallManifest(path=manifest)
+        c = FileCopier()
+        m.populate_registry(c)
+        c.copy(dest)
+
+        with open(destfile, 'rt') as fh:
+            self.assertEqual(fh.read(), 'SOURCE\nINCLUDE MODIFIED\n')
 
 if __name__ == '__main__':
     mozunit.main()