Bug 836218 - Always copy files instead of skipping when destination is newer when doing l10n-repacks. r=ted
authorMike Hommey <mh+mozilla@glandium.org>
Sun, 03 Feb 2013 07:20:24 +0100
changeset 120607 5f21061637d59d26776fa8cb8cb975326bb50a1a
parent 120606 d949c8ee39da1edc021415377c4a4fc74caa530e
child 120608 ba515e203813b34f6feaaa4ad0d0a971f828e7f5
push id24259
push usermh@glandium.org
push dateSun, 03 Feb 2013 06:23:08 +0000
treeherdermozilla-central@ba515e203813 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs836218
milestone21.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 836218 - Always copy files instead of skipping when destination is newer when doing l10n-repacks. r=ted
python/mozbuild/mozpack/copier.py
python/mozbuild/mozpack/files.py
python/mozbuild/mozpack/packager/unpack.py
python/mozbuild/mozpack/test/test_files.py
python/mozbuild/mozpack/unify.py
toolkit/mozapps/installer/l10n-repack.py
toolkit/mozapps/installer/packager.py
--- a/python/mozbuild/mozpack/copier.py
+++ b/python/mozbuild/mozpack/copier.py
@@ -125,17 +125,17 @@ class FileRegistry(object):
         return self._files.iteritems()
 
 
 class FileCopier(FileRegistry):
     '''
     FileRegistry with the ability to copy the registered files to a separate
     directory.
     '''
-    def copy(self, destination):
+    def copy(self, destination, skip_if_older=True):
         '''
         Copy all registered files to the given destination path. The given
         destination can be an existing directory, or not exist at all. It
         can't be e.g. a file.
         The copy process acts a bit like rsync: files are not copied when they
         don't need to (see mozpack.files for details on file.copy), and files
         existing in the destination directory that aren't registered are
         removed.
@@ -143,17 +143,17 @@ class FileCopier(FileRegistry):
         assert isinstance(destination, basestring)
         assert not os.path.exists(destination) or os.path.isdir(destination)
         destination = os.path.normpath(destination)
         dest_files = set()
         for path, file in self:
             destfile = os.path.normpath(os.path.join(destination, path))
             dest_files.add(destfile)
             ensure_parent_dir(destfile)
-            file.copy(destfile)
+            file.copy(destfile, skip_if_older)
 
         actual_dest_files = set()
         for root, dirs, files in os.walk(destination):
             for f in files:
                 actual_dest_files.add(os.path.normpath(os.path.join(root, f)))
         for f in actual_dest_files - dest_files:
             os.remove(f)
         for root, dirs, files in os.walk(destination):
@@ -171,17 +171,17 @@ class Jarrer(FileRegistry, BaseFile):
         Create a Jarrer instance. See mozpack.mozjar.JarWriter documentation
         for details on the compress and optimize arguments.
         '''
         self.compress = compress
         self.optimize = optimize
         self._preload = []
         FileRegistry.__init__(self)
 
-    def copy(self, dest):
+    def copy(self, dest, skip_if_older=True):
         '''
         Pack all registered files in the given destination jar. The given
         destination jar may be a path to jar file, or a Dest instance for
         a jar file.
         If the destination jar file exists, its (compressed) contents are used
         instead of the registered BaseFile instances when appropriate.
         '''
         class DeflaterDest(Dest):
@@ -229,17 +229,17 @@ class Jarrer(FileRegistry, BaseFile):
 
         with JarWriter(fileobj=dest, compress=self.compress,
                        optimize=self.optimize) as jar:
             for path, file in self:
                 if path in old_contents:
                     deflater = DeflaterDest(old_contents[path], self.compress)
                 else:
                     deflater = DeflaterDest(compress=self.compress)
-                file.copy(deflater)
+                file.copy(deflater, skip_if_older)
                 jar.add(path, deflater.deflater)
             if self._preload:
                 jar.preload(self._preload)
 
     def open(self):
         raise RuntimeError('unsupported')
 
     def preload(self, paths):
--- a/python/mozbuild/mozpack/files.py
+++ b/python/mozbuild/mozpack/files.py
@@ -56,38 +56,39 @@ class Dest(object):
 
 
 class BaseFile(object):
     '''
     Base interface and helper for file copying. Derived class may implement
     their own copy function, or rely on BaseFile.copy using the open() member
     function and/or the path property.
     '''
-    def copy(self, dest):
+    def copy(self, dest, skip_if_older=True):
         '''
         Copy the BaseFile content to the destination given as a string or a
         Dest instance. Avoids replacing existing files if the BaseFile content
         matches that of the destination, or in case of plain files, if the
-        destination is newer than the original file.
+        destination is newer than the original file. This latter behaviour is
+        disabled when skip_if_older is False.
         Returns whether a copy was actually performed (True) or not (False).
         '''
         if isinstance(dest, basestring):
             dest = Dest(dest)
         else:
             assert isinstance(dest, Dest)
 
         can_skip_content_check = False
         if not dest.exists():
             can_skip_content_check = True
         elif getattr(self, 'path', None) and getattr(dest, 'path', None):
             # os.path.getmtime returns a result in seconds with precision up to
             # the microsecond. But microsecond is too precise because
             # shutil.copystat only copies milliseconds, and seconds is not
             # enough precision.
-            if int(os.path.getmtime(self.path) * 1000) \
+            if skip_if_older and int(os.path.getmtime(self.path) * 1000) \
                     <= int(os.path.getmtime(dest.path) * 1000):
                 return False
             elif os.path.getsize(self.path) != os.path.getsize(dest.path):
                 can_skip_content_check = True
 
         if can_skip_content_check:
             if getattr(self, 'path', None) and getattr(dest, 'path', None):
                 shutil.copy2(self.path, dest.path)
@@ -134,19 +135,19 @@ class File(BaseFile):
         self.path = path
 
 
 class ExecutableFile(File):
     '''
     File class for executable and library files on OS/2, OS/X and ELF systems.
     (see mozpack.executables.is_executable documentation).
     '''
-    def copy(self, dest):
+    def copy(self, dest, skip_if_older=True):
         assert isinstance(dest, basestring)
-        File.copy(self, dest)
+        File.copy(self, dest, skip_if_older)
         try:
             if may_strip(dest):
                 strip(dest)
             if may_elfhack(dest):
                 elfhack(dest)
         except ErrorMessage:
             os.remove(dest)
             raise
@@ -199,22 +200,23 @@ class XPTFile(GeneratedFile):
     def remove(self, xpt):
         '''
         Remove the given XPT file (as a BaseFile instance) from the list of
         XPTs to link.
         '''
         assert isinstance(xpt, BaseFile)
         self._files.remove(xpt)
 
-    def copy(self, dest):
+    def copy(self, dest, skip_if_older=True):
         '''
         Link the registered XPTs and place the resulting linked XPT at the
         destination given as a string or a Dest instance. Avoids an expensive
         XPT linking if the interfaces in an existing destination match those of
         the individual XPTs to link.
+        skip_if_older is ignored.
         '''
         if isinstance(dest, basestring):
             dest = Dest(dest)
         assert isinstance(dest, Dest)
 
         from xpt import xpt_link, Typelib, Interface
         all_typelibs = [Typelib.read(f.open()) for f in self._files]
         if dest.exists():
--- a/python/mozbuild/mozpack/packager/unpack.py
+++ b/python/mozbuild/mozpack/packager/unpack.py
@@ -166,9 +166,9 @@ def unpack(source):
     '''
     copier = FileCopier()
     finder = UnpackFinder(source)
     packager = SimplePackager(FlatFormatter(copier))
     for p, f in finder.find('*'):
         if mozpack.path.split(p)[0] not in STARTUP_CACHE_PATHS:
             packager.add(p, f)
     packager.close()
-    copier.copy(source)
+    copier.copy(source, skip_if_older=False)
--- a/python/mozbuild/mozpack/test/test_files.py
+++ b/python/mozbuild/mozpack/test/test_files.py
@@ -205,16 +205,20 @@ class TestFile(TestWithTmpDir):
         self.assertEqual('test', open(dest, 'rb').read())
 
         # Double check that under conditions where a copy occurs, we would get
         # an exception.
         time = os.path.getmtime(src) - 1
         os.utime(dest, (time, time))
         self.assertRaises(RuntimeError, f.copy, DestNoWrite(dest))
 
+        # skip_if_older=False is expected to force a copy in this situation.
+        f.copy(dest, skip_if_older=False)
+        self.assertEqual('fooo', open(dest, 'rb').read())
+
 
 class TestGeneratedFile(TestWithTmpDir):
     def test_generated_file(self):
         '''
         Check that GeneratedFile.copy yields the proper content in the
         destination file in all situations that trigger different code paths
         (see TestFile.test_file)
         '''
--- a/python/mozbuild/mozpack/unify.py
+++ b/python/mozbuild/mozpack/unify.py
@@ -47,17 +47,22 @@ class UnifiedExecutableFile(BaseFile):
     def __init__(self, path1, path2):
         '''
         Initialize a UnifiedExecutableFile with the path to both non-fat Mach-O
         executables to be unified.
         '''
         self.path1 = path1
         self.path2 = path2
 
-    def copy(self, dest):
+    def copy(self, dest, skip_if_older=True):
+        '''
+        Create a fat executable from the two Mach-O executable given when
+        creating the instance.
+        skip_if_older is ignored.
+        '''
         assert isinstance(dest, basestring)
         tmpfiles = []
         try:
             for p in [self.path1, self.path2]:
                 fd, f = mkstemp()
                 os.close(fd)
                 tmpfiles.append(f)
                 shutil.copy2(p, f)
--- a/toolkit/mozapps/installer/l10n-repack.py
+++ b/toolkit/mozapps/installer/l10n-repack.py
@@ -175,17 +175,17 @@ def repack(source, l10n, non_resources=[
             if not formatter.contains(p):
                 formatter.add(p, f)
 
     # Transplant jar preloading information.
     for path, log in finder.jarlogs.iteritems():
         assert isinstance(copier[path], Jarrer)
         copier[path].preload([l.replace(locale, l10n_locale) for l in log])
 
-    copier.copy(source)
+    copier.copy(source, skip_if_older=False)
     generate_precomplete(source)
 
 
 def main():
     parser = ArgumentParser()
     parser.add_argument('build',
                         help='Directory containing the build to repack')
     parser.add_argument('l10n',
--- a/toolkit/mozapps/installer/packager.py
+++ b/toolkit/mozapps/installer/packager.py
@@ -87,22 +87,22 @@ class ToolLauncher(object):
 
 launcher = ToolLauncher()
 
 
 class LibSignFile(File):
     '''
     File class for shlibsign signatures.
     '''
-    def copy(self, dest):
+    def copy(self, dest, skip_if_older=True):
         assert isinstance(dest, basestring)
         # os.path.getmtime returns a result in seconds with precision up to the
         # microsecond. But microsecond is too precise because shutil.copystat
         # only copies milliseconds, and seconds is not enough precision.
-        if os.path.exists(dest) and \
+        if os.path.exists(dest) and skip_if_older and \
                 int(os.path.getmtime(self.path) * 1000) <= \
                 int(os.path.getmtime(dest) * 1000):
             return False
         if launcher.launch(['shlibsign', '-v', '-o', dest, '-i', self.path]):
             errors.fatal('Error while signing %s' % self.path)
 
 
 def precompile_cache(formatter, source_path, gre_path, app_path):