Bug 935987 - Part 2: Add preprocessed files to mozpack.files; r=gps
authorBrian O'Keefe <bokeefe@alum.wpi.edu>
Wed, 06 Nov 2013 14:46:05 -0500
changeset 164109 cbb166c4a60d198c0824eb05687ce1aa32386ba6
parent 164108 8526c7a387617c0d9db00c873e2b6bc3a0fbd5a4
child 164110 48289161bc17e4c87c061c63b5233eba07644704
push id26026
push userphilringnalda@gmail.com
push dateSat, 18 Jan 2014 23:17:27 +0000
treeherdermozilla-central@61fd0f987cf2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs935987
milestone29.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 935987 - Part 2: Add preprocessed files to mozpack.files; r=gps
python/mozbuild/mozpack/files.py
python/mozbuild/mozpack/test/test_files.py
--- a/python/mozbuild/mozpack/files.py
+++ b/python/mozbuild/mozpack/files.py
@@ -3,16 +3,19 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import errno
 import os
 import re
 import shutil
 import stat
 import uuid
+import mozbuild.makeutil as makeutil
+from mozbuild.preprocessor import Preprocessor
+from mozbuild.util import FileAvoidWrite
 from mozpack.executables import (
     is_executable,
     may_strip,
     strip,
     may_elfhack,
     elfhack,
 )
 from mozpack.chrome.manifest import ManifestEntry
@@ -35,16 +38,20 @@ class Dest(object):
       read from it.
     - a call to write() after a read() will re-open the underlying file,
       emptying it, and write to it.
     '''
     def __init__(self, path):
         self.path = path
         self.mode = None
 
+    @property
+    def name(self):
+        return self.path
+
     def read(self, length=-1):
         if self.mode != 'r':
             self.file = open(self.path, 'rb')
             self.mode = 'r'
         return self.file.read(length)
 
     def write(self, data):
         if self.mode != 'w':
@@ -62,16 +69,46 @@ class Dest(object):
 
 
 class BaseFile(object):
     '''
     Base interface and helper for file copying. Derived class may implement
     their own copy function, or rely on BaseFile.copy using the open() member
     function and/or the path property.
     '''
+    @staticmethod
+    def is_older(first, second):
+        '''
+        Compares the modification time of two files, and returns whether the
+        ``first`` file is older than the ``second`` file.
+        '''
+        # os.path.getmtime returns a result in seconds with precision up to
+        # the microsecond. But microsecond is too precise because
+        # shutil.copystat only copies milliseconds, and seconds is not
+        # enough precision.
+        return int(os.path.getmtime(first) * 1000) \
+                <= int(os.path.getmtime(second) * 1000)
+
+    @staticmethod
+    def any_newer(dest, inputs):
+        '''
+        Compares the modification time of ``dest`` to multiple input files, and
+        returns whether any of the ``inputs`` is newer (has a later mtime) than
+        ``dest``.
+        '''
+        # os.path.getmtime returns a result in seconds with precision up to
+        # the microsecond. But microsecond is too precise because
+        # shutil.copystat only copies milliseconds, and seconds is not
+        # enough precision.
+        dest_mtime = int(os.path.getmtime(dest) * 1000)
+        for input in inputs:
+            if dest_mtime < int(os.path.getmtime(input) * 1000):
+                return True
+        return False
+
     def copy(self, dest, skip_if_older=True):
         '''
         Copy the BaseFile content to the destination given as a string or a
         Dest instance. Avoids replacing existing files if the BaseFile content
         matches that of the destination, or in case of plain files, if the
         destination is newer than the original file. This latter behaviour is
         disabled when skip_if_older is False.
         Returns whether a copy was actually performed (True) or not (False).
@@ -80,22 +117,17 @@ class BaseFile(object):
             dest = Dest(dest)
         else:
             assert isinstance(dest, Dest)
 
         can_skip_content_check = False
         if not dest.exists():
             can_skip_content_check = True
         elif getattr(self, 'path', None) and getattr(dest, 'path', None):
-            # os.path.getmtime returns a result in seconds with precision up to
-            # the microsecond. But microsecond is too precise because
-            # shutil.copystat only copies milliseconds, and seconds is not
-            # enough precision.
-            if skip_if_older and int(os.path.getmtime(self.path) * 1000) \
-                    <= int(os.path.getmtime(dest.path) * 1000):
+            if skip_if_older and BaseFile.is_older(self.path, dest.path):
                 return False
             elif os.path.getsize(self.path) != os.path.getsize(dest.path):
                 can_skip_content_check = True
 
         if can_skip_content_check:
             if getattr(self, 'path', None) and getattr(dest, 'path', None):
                 shutil.copy2(self.path, dest.path)
             else:
@@ -287,16 +319,86 @@ class ExistingFile(BaseFile):
         if not self.required:
             return
 
         if not dest.exists():
             errors.fatal("Required existing file doesn't exist: %s" %
                 dest.path)
 
 
+class PreprocessedFile(BaseFile):
+    '''
+    File class for a file that is preprocessed. PreprocessedFile.copy() runs
+    the preprocessor on the file to create the output.
+    '''
+    def __init__(self, path, depfile_path, marker, defines, extra_depends=None):
+        self.path = path
+        self.depfile = depfile_path
+        self.marker = marker
+        self.defines = defines
+        self.extra_depends = list(extra_depends or [])
+
+    def copy(self, dest, skip_if_older=True):
+        '''
+        Invokes the preprocessor to create the destination file.
+        '''
+        if isinstance(dest, basestring):
+            dest = Dest(dest)
+        else:
+            assert isinstance(dest, Dest)
+
+        # We have to account for the case where the destination exists and is a
+        # symlink to something. Since we know the preprocessor is certainly not
+        # going to create a symlink, we can just remove the existing one. If the
+        # destination is not a symlink, we leave it alone, since we're going to
+        # overwrite its contents anyway.
+        # If symlinks aren't supported at all, we can skip this step.
+        if hasattr(os, 'symlink'):
+            if os.path.islink(dest.path):
+                os.remove(dest.path)
+
+        pp_deps = set(self.extra_depends)
+
+        # If a dependency file was specified, and it exists, add any
+        # dependencies from that file to our list.
+        if self.depfile and os.path.exists(self.depfile):
+            target = mozpack.path.normpath(dest.name)
+            with open(self.depfile, 'rb') as fileobj:
+                for rule in makeutil.read_dep_makefile(fileobj):
+                    if target in rule.targets():
+                        pp_deps.update(rule.dependencies())
+
+        skip = False
+        if dest.exists() and skip_if_older:
+            # If a dependency file was specified, and it doesn't exist,
+            # assume that the preprocessor needs to be rerun. That will
+            # regenerate the dependency file.
+            if self.depfile and not os.path.exists(self.depfile):
+                skip = False
+            else:
+                skip = not BaseFile.any_newer(dest.path, pp_deps)
+
+        if skip:
+            return False
+
+        deps_out = None
+        if self.depfile:
+            deps_out = FileAvoidWrite(self.depfile)
+        pp = Preprocessor(defines=self.defines, marker=self.marker)
+
+        with open(self.path, 'rU') as input:
+            pp.processFile(input=input, output=dest, depfile=deps_out)
+
+        dest.close()
+        if self.depfile:
+            deps_out.close()
+
+        return True
+
+
 class GeneratedFile(BaseFile):
     '''
     File class for content with no previous existence on the filesystem.
     '''
     def __init__(self, content):
         self.content = content
 
     def open(self):
--- a/python/mozbuild/mozpack/test/test_files.py
+++ b/python/mozbuild/mozpack/test/test_files.py
@@ -11,16 +11,17 @@ from mozpack.files import (
     Dest,
     ExistingFile,
     FileFinder,
     File,
     GeneratedFile,
     JarFinder,
     ManifestFile,
     MinifiedProperties,
+    PreprocessedFile,
     XPTFile,
 )
 from mozpack.mozjar import (
     JarReader,
     JarWriter,
 )
 from mozpack.chrome.manifest import (
     ManifestContent,
@@ -322,16 +323,138 @@ class TestAbsoluteSymlinkFile(TestWithTm
         self.assertEqual(link, source)
 
         s = AbsoluteSymlinkFile(source)
         self.assertFalse(s.copy(dest))
 
         link = os.readlink(dest)
         self.assertEqual(link, source)
 
+class TestPreprocessedFile(TestWithTmpDir):
+    def test_preprocess(self):
+        '''
+        Test that copying the file invokes the preprocessor
+        '''
+        src = self.tmppath('src')
+        dest = self.tmppath('dest')
+
+        with open(src, 'wb') as tmp:
+            tmp.write('#ifdef FOO\ntest\n#endif')
+
+        f = PreprocessedFile(src, depfile_path=None, marker='#', defines={'FOO': True})
+        self.assertTrue(f.copy(dest))
+
+        self.assertEqual('test\n', open(dest, 'rb').read())
+
+    def test_preprocess_file_no_write(self):
+        '''
+        Test various conditions where PreprocessedFile.copy is expected not to
+        write in the destination file.
+        '''
+        src = self.tmppath('src')
+        dest = self.tmppath('dest')
+        depfile = self.tmppath('depfile')
+
+        with open(src, 'wb') as tmp:
+            tmp.write('#ifdef FOO\ntest\n#endif')
+
+        # Initial copy
+        f = PreprocessedFile(src, depfile_path=depfile, marker='#', defines={'FOO': True})
+        self.assertTrue(f.copy(dest))
+
+        # Ensure subsequent copies won't trigger writes
+        self.assertFalse(f.copy(DestNoWrite(dest)))
+        self.assertEqual('test\n', open(dest, 'rb').read())
+
+        # When the source file is older than the destination file, even with
+        # different content, no copy should occur.
+        with open(src, 'wb') as tmp:
+            tmp.write('#ifdef FOO\nfooo\n#endif')
+        time = os.path.getmtime(dest) - 1
+        os.utime(src, (time, time))
+        self.assertFalse(f.copy(DestNoWrite(dest)))
+        self.assertEqual('test\n', open(dest, 'rb').read())
+
+        # skip_if_older=False is expected to force a copy in this situation.
+        self.assertTrue(f.copy(dest, skip_if_older=False))
+        self.assertEqual('fooo\n', open(dest, 'rb').read())
+
+    def test_preprocess_file_dependencies(self):
+        '''
+        Test that the preprocess runs if the dependencies of the source change
+        '''
+        src = self.tmppath('src')
+        dest = self.tmppath('dest')
+        incl = self.tmppath('incl')
+        deps = self.tmppath('src.pp')
+
+        with open(src, 'wb') as tmp:
+            tmp.write('#ifdef FOO\ntest\n#endif')
+
+        with open(incl, 'wb') as tmp:
+            tmp.write('foo bar')
+
+        # Initial copy
+        f = PreprocessedFile(src, depfile_path=deps, marker='#', defines={'FOO': True})
+        self.assertTrue(f.copy(dest))
+
+        # Update the source so it #includes the include file.
+        with open(src, 'wb') as tmp:
+            tmp.write('#include incl\n')
+        time = os.path.getmtime(dest) + 1
+        os.utime(src, (time, time))
+        self.assertTrue(f.copy(dest))
+        self.assertEqual('foo bar', open(dest, 'rb').read())
+
+        # If one of the dependencies changes, the file should be updated. The
+        # mtime of the dependency is set after the destination file, to avoid
+        # both files having the same time.
+        with open(incl, 'wb') as tmp:
+            tmp.write('quux')
+        time = os.path.getmtime(dest) + 1
+        os.utime(incl, (time, time))
+        self.assertTrue(f.copy(dest))
+        self.assertEqual('quux', open(dest, 'rb').read())
+
+        # Perform one final copy to confirm that we don't run the preprocessor
+        # again. We update the mtime of the destination so it's newer than the
+        # input files. This would "just work" if we weren't changing
+        time = os.path.getmtime(incl) + 1
+        os.utime(dest, (time, time))
+        self.assertFalse(f.copy(DestNoWrite(dest)))
+
+    def test_replace_symlink(self):
+        '''
+        Test that if the destination exists, and is a symlink, the target of
+        the symlink is not overwritten by the preprocessor output.
+        '''
+        if not self.symlink_supported:
+            return
+
+        source = self.tmppath('source')
+        dest = self.tmppath('dest')
+        pp_source = self.tmppath('pp_in')
+        deps = self.tmppath('deps')
+
+        with open(source, 'a'):
+            pass
+
+        os.symlink(source, dest)
+        self.assertTrue(os.path.islink(dest))
+
+        with open(pp_source, 'wb') as tmp:
+            tmp.write('#define FOO\nPREPROCESSED')
+
+        f = PreprocessedFile(pp_source, depfile_path=deps, marker='#',
+            defines={'FOO': True})
+        self.assertTrue(f.copy(dest))
+
+        self.assertEqual('PREPROCESSED', open(dest, 'rb').read())
+        self.assertFalse(os.path.islink(dest))
+        self.assertEqual('', open(source, 'rb').read())
 
 class TestExistingFile(TestWithTmpDir):
     def test_required_missing_dest(self):
         with self.assertRaisesRegexp(ErrorMessage, 'Required existing file'):
             f = ExistingFile(required=True)
             f.copy(self.tmppath('dest'))
 
     def test_required_existing_dest(self):