Bug 890097 - Part 3: Capture detailed information for FileCopier actions; r=glandium
authorGregory Szorc <gps@mozilla.com>
Tue, 23 Jul 2013 14:37:04 -0700
changeset 152002 a0fa8c9992a5651ea22610cf9db928ba2a4d1bab
parent 152001 d27457347dd6249b694432b6698a3f07b31194bb
child 152003 fe626c54e50c452fc1508177e714050159d6fd46
push id2859
push userakeybl@mozilla.com
push dateMon, 16 Sep 2013 19:14:59 +0000
treeherdermozilla-beta@87d3c51cd2bf [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs890097
milestone25.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 890097 - Part 3: Capture detailed information for FileCopier actions; r=glandium
python/mozbuild/mozpack/copier.py
python/mozbuild/mozpack/test/test_copier.py
python/mozbuild/mozpack/test/test_manifests.py
--- a/python/mozbuild/mozpack/copier.py
+++ b/python/mozbuild/mozpack/copier.py
@@ -6,17 +6,16 @@ import os
 from mozpack.errors import errors
 from mozpack.files import (
     BaseFile,
     Dest,
 )
 import mozpack.path
 import errno
 from collections import (
-    namedtuple,
     OrderedDict,
 )
 
 
 def ensure_parent_dir(file):
     '''Ensures the directory parent to the given file exists'''
     dir = os.path.dirname(file)
     if not dir:
@@ -129,18 +128,40 @@ class FileRegistry(object):
         '''
         Iterate over all (path, BaseFile instance) pairs from the container.
             for path, file in registry:
                 (...)
         '''
         return self._files.iteritems()
 
 
-FileCopyResult = namedtuple('FileCopyResult', ['removed_files_count',
-    'removed_directories_count'])
+class FileCopyResult(object):
+    """Represents results of a FileCopier.copy operation."""
+
+    def __init__(self):
+        self.updated_files = set()
+        self.existing_files = set()
+        self.removed_files = set()
+        self.removed_directories = set()
+
+    @property
+    def updated_files_count(self):
+        return len(self.updated_files)
+
+    @property
+    def existing_files_count(self):
+        return len(self.existing_files)
+
+    @property
+    def removed_files_count(self):
+        return len(self.removed_files)
+
+    @property
+    def removed_directories_count(self):
+        return len(self.removed_directories)
 
 
 class FileCopier(FileRegistry):
     '''
     FileRegistry with the ability to copy the registered files to a separate
     directory.
     '''
     def copy(self, destination, skip_if_older=True):
@@ -152,54 +173,54 @@ class FileCopier(FileRegistry):
         don't need to (see mozpack.files for details on file.copy), and files
         existing in the destination directory that aren't registered are
         removed.
 
         Returns a FileCopyResult that details what changed.
         '''
         assert isinstance(destination, basestring)
         assert not os.path.exists(destination) or os.path.isdir(destination)
+        result = FileCopyResult()
         destination = os.path.normpath(destination)
         dest_files = set()
         for path, file in self:
             destfile = os.path.normpath(os.path.join(destination, path))
             dest_files.add(destfile)
             ensure_parent_dir(destfile)
-            file.copy(destfile, skip_if_older)
+            if file.copy(destfile, skip_if_older):
+                result.updated_files.add(destfile)
+            else:
+                result.existing_files.add(destfile)
 
         actual_dest_files = set()
         for root, dirs, files in os.walk(destination):
             for f in files:
                 actual_dest_files.add(os.path.normpath(os.path.join(root, f)))
 
-        file_remove_count = 0
-        directory_remove_count = 0
-
         for f in actual_dest_files - dest_files:
             # Windows requires write access to remove files.
             if os.name == 'nt' and not os.access(f, os.W_OK):
                 # It doesn't matter what we set the permissions to since we
                 # will remove this file shortly.
                 os.chmod(f, 0600)
 
             os.remove(f)
-            file_remove_count += 1
+            result.removed_files.add(f)
 
         for root, dirs, files in os.walk(destination):
             if files or dirs:
                 continue
 
             # Like files, permissions may not allow deletion. So, ensure write
             # access is in place before attempting delete.
             os.chmod(root, 0700)
             os.removedirs(root)
-            directory_remove_count += 1
+            result.removed_directories.add(root)
 
-        return FileCopyResult(removed_files_count=file_remove_count,
-            removed_directories_count=directory_remove_count)
+        return result
 
 
 class FilePurger(FileCopier):
     """A variation of FileCopier that is used to purge untracked files.
 
     Callers create an instance then call .add() to register files/paths that
     should exist. Once the canonical set of files that may exist is defined,
     .purge() is called against a target directory. All files and empty
--- a/python/mozbuild/mozpack/test/test_copier.py
+++ b/python/mozbuild/mozpack/test/test_copier.py
@@ -110,26 +110,34 @@ class TestFileCopier(TestWithTmpDir):
         copier = FileCopier()
         copier.add('foo/bar', GeneratedFile('foobar'))
         copier.add('foo/qux', GeneratedFile('fooqux'))
         copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz'))
         copier.add('bar', GeneratedFile('bar'))
         copier.add('qux/foo', GeneratedFile('quxfoo'))
         copier.add('qux/bar', GeneratedFile(''))
 
-        copier.copy(self.tmpdir)
+        result = copier.copy(self.tmpdir)
         self.assertEqual(self.all_files(self.tmpdir), set(copier.paths()))
         self.assertEqual(self.all_dirs(self.tmpdir),
                          set(['foo/deep/nested/directory', 'qux']))
 
+        self.assertEqual(result.updated_files, set(self.tmppath(p) for p in
+            self.all_files(self.tmpdir)))
+        self.assertEqual(result.existing_files, set())
+        self.assertEqual(result.removed_files, set())
+        self.assertEqual(result.removed_directories, set())
+
         copier.remove('foo')
         copier.add('test', GeneratedFile('test'))
-        copier.copy(self.tmpdir)
+        result = copier.copy(self.tmpdir)
         self.assertEqual(self.all_files(self.tmpdir), set(copier.paths()))
         self.assertEqual(self.all_dirs(self.tmpdir), set(['qux']))
+        self.assertEqual(result.removed_files, set(self.tmppath(p) for p in
+            ('foo/bar', 'foo/qux', 'foo/deep/nested/directory/file')))
 
     def test_permissions(self):
         """Ensure files without write permission can be deleted."""
         with open(self.tmppath('dummy'), 'a'):
             pass
 
         p = self.tmppath('no_perms')
         with open(p, 'a'):
@@ -165,16 +173,18 @@ class TestFilePurger(TestWithTmpDir):
             pass
 
         self.assertTrue(os.path.exists(existing))
         self.assertTrue(os.path.exists(extra))
 
         purger = FilePurger()
         purger.add('existing')
         result = purger.purge(self.tmpdir)
+        self.assertEqual(result.removed_files, set(self.tmppath(p) for p in
+            ('extra', 'dir/foo')))
         self.assertEqual(result.removed_files_count, 2)
         self.assertEqual(result.removed_directories_count, 1)
 
         self.assertTrue(os.path.exists(existing))
         self.assertFalse(os.path.exists(extra))
         self.assertFalse(os.path.exists(empty_dir))
 
 
--- a/python/mozbuild/mozpack/test/test_manifests.py
+++ b/python/mozbuild/mozpack/test/test_manifests.py
@@ -153,24 +153,32 @@ class TestInstallManifest(TestWithTmpDir
             fh.write('copy!')
 
         with open(self.tmppath('dest/e_dest'), 'a'):
             pass
 
         m = self._get_test_manifest()
         c = FileCopier()
         m.populate_registry(c)
-        c.copy(dest)
+        result = c.copy(dest)
 
         self.assertTrue(os.path.exists(self.tmppath('dest/s_dest')))
         self.assertTrue(os.path.exists(self.tmppath('dest/c_dest')))
         self.assertTrue(os.path.exists(self.tmppath('dest/e_dest')))
         self.assertFalse(os.path.exists(to_delete))
 
         with open(self.tmppath('dest/s_dest'), 'rt') as fh:
             self.assertEqual(fh.read(), 'symlink!')
 
         with open(self.tmppath('dest/c_dest'), 'rt') as fh:
             self.assertEqual(fh.read(), 'copy!')
 
+        self.assertEqual(result.updated_files, set(self.tmppath(p) for p in (
+            'dest/s_dest', 'dest/c_dest', 'dest/e_dest')))
+        self.assertEqual(result.existing_files, set())
+        self.assertEqual(result.removed_files, {to_delete})
+        self.assertEqual(result.removed_directories, set())
+
+
+
 
 if __name__ == '__main__':
     mozunit.main()