Bug 1240134 - Use the UnpackFinder when extracting from archives for an artifact build. r=glandium
authorChris Manchester <cmanchester@mozilla.com>
Wed, 17 Aug 2016 15:02:31 -0700
changeset 310040 91dfed0af1dc9aa2dc5015414acd6380caeb9029
parent 310039 0c2c15236f5b1708c55520b0fc97d8e386db2b9e
child 310041 98bc41e5c777e70f1ae9772032860f2122c7c3b3
push id30576
push userryanvm@gmail.com
push dateFri, 19 Aug 2016 13:53:39 +0000
treeherdermozilla-central@74f332c38a69 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1240134
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1240134 - Use the UnpackFinder when extracting from archives for an artifact build. r=glandium MozReview-Commit-ID: C5Y9oC6ywwP
python/mozbuild/mozbuild/artifacts.py
--- a/python/mozbuild/mozbuild/artifacts.py
+++ b/python/mozbuild/mozbuild/artifacts.py
@@ -63,21 +63,25 @@ import zipfile
 import pylru
 import taskcluster
 
 from mozbuild.util import (
     ensureParentDir,
     FileAvoidWrite,
 )
 import mozinstall
-from mozpack.files import FileFinder
+from mozpack.files import (
+    JarFinder,
+    TarFinder,
+)
 from mozpack.mozjar import (
     JarReader,
     JarWriter,
 )
+from mozpack.packager.unpack import UnpackFinder
 import mozpack.path as mozpath
 from mozregression.download_manager import (
     DownloadManager,
 )
 from mozregression.persist_limit import (
     PersistLimit,
 )
 
@@ -179,32 +183,42 @@ class ArtifactJob(object):
                     writer.add(destpath.encode('utf-8'), reader[filename], mode=mode)
                     added_entry = True
 
         if not added_entry:
             raise ValueError('Archive format changed! No pattern from "{patterns}"'
                              'matched an archive path.'.format(
                                  patterns=LinuxArtifactJob.test_artifact_patterns))
 
+
 class AndroidArtifactJob(ArtifactJob):
+
+    package_artifact_patterns = {
+        'application.ini',
+        'platform.ini',
+        '**/*.so',
+    }
+
     def process_artifact(self, filename, processed_filename):
         # Extract all .so files into the root, which will get copied into dist/bin.
         with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
-            for f in JarReader(filename):
-                if not f.filename.endswith('.so') and \
-                   not f.filename in ('platform.ini', 'application.ini'):
+            for p, f in UnpackFinder(JarFinder(filename, JarReader(filename))):
+                if not any(mozpath.match(p, pat) for pat in self.package_artifact_patterns):
                     continue
 
-                basename = os.path.basename(f.filename)
+                dirname, basename = os.path.split(p)
                 self.log(logging.INFO, 'artifact',
                     {'basename': basename},
                    'Adding {basename} to processed archive')
 
-                basename = mozpath.join('bin', basename)
-                writer.add(basename.encode('utf-8'), f)
+                basedir = 'bin'
+                if not basename.endswith('.so'):
+                    basedir = mozpath.join('bin', dirname.lstrip('assets/'))
+                basename = mozpath.join(basedir, basename)
+                writer.add(basename.encode('utf-8'), f.open())
 
 
 class LinuxArtifactJob(ArtifactJob):
 
     package_artifact_patterns = {
         'firefox/application.ini',
         'firefox/crashreporter',
         'firefox/dependentlibs.list',
@@ -216,31 +230,28 @@ class LinuxArtifactJob(ArtifactJob):
         'firefox/**/*.so',
     }
 
     def process_package_artifact(self, filename, processed_filename):
         added_entry = False
 
         with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
             with tarfile.open(filename) as reader:
-                for f in reader:
-                    if not f.isfile():
-                        continue
-
-                    if not any(mozpath.match(f.name, p) for p in self.package_artifact_patterns):
+                for p, f in UnpackFinder(TarFinder(filename, reader)):
+                    if not any(mozpath.match(p, pat) for pat in self.package_artifact_patterns):
                         continue
 
                     # We strip off the relative "firefox/" bit from the path,
                     # but otherwise preserve it.
                     destpath = mozpath.join('bin',
-                                            mozpath.relpath(f.name, "firefox"))
+                                            mozpath.relpath(p, "firefox"))
                     self.log(logging.INFO, 'artifact',
                              {'destpath': destpath},
                              'Adding {destpath} to processed archive')
-                    writer.add(destpath.encode('utf-8'), reader.extractfile(f), mode=f.mode)
+                    writer.add(destpath.encode('utf-8'), f.open(), mode=f.mode)
                     added_entry = True
 
         if not added_entry:
             raise ValueError('Archive format changed! No pattern from "{patterns}" '
                              'matched an archive path.'.format(
                                  patterns=LinuxArtifactJob.package_artifact_patterns))
 
 
@@ -306,34 +317,34 @@ class MacArtifactJob(ArtifactJob):
                 # 'firefox',
                 'gmp-clearkey/0.1/libclearkey.dylib',
                 # 'gmp-fake/1.0/libfake.dylib',
                 # 'gmp-fakeopenh264/1.0/libfakeopenh264.dylib',
             ])
 
             with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
                 root, paths = paths_no_keep_path
-                finder = FileFinder(mozpath.join(source, root))
+                finder = UnpackFinder(mozpath.join(source, root))
                 for path in paths:
                     for p, f in finder.find(path):
                         self.log(logging.INFO, 'artifact',
-                            {'path': path},
+                            {'path': p},
                             'Adding {path} to processed archive')
                         destpath = mozpath.join('bin', os.path.basename(p))
-                        writer.add(destpath.encode('utf-8'), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode)
+                        writer.add(destpath.encode('utf-8'), f, mode=f.mode)
 
                 root, paths = paths_keep_path
-                finder = FileFinder(mozpath.join(source, root))
+                finder = UnpackFinder(mozpath.join(source, root))
                 for path in paths:
                     for p, f in finder.find(path):
                         self.log(logging.INFO, 'artifact',
-                            {'path': path},
+                            {'path': p},
                             'Adding {path} to processed archive')
                         destpath = mozpath.join('bin', p)
-                        writer.add(destpath.encode('utf-8'), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode)
+                        writer.add(destpath.encode('utf-8'), f.open(), mode=f.mode)
 
         finally:
             try:
                 shutil.rmtree(tempdir)
             except (OSError, IOError):
                 self.log(logging.WARN, 'artifact',
                     {'tempdir': tempdir},
                     'Unable to delete {tempdir}')
@@ -359,27 +370,27 @@ class WinArtifactJob(ArtifactJob):
         ('bin/ssltunnel.exe', ('bin', 'bin')),
         ('bin/xpcshell.exe', ('bin', 'bin')),
         ('bin/plugins/*', ('bin/plugins', 'plugins'))
     }
 
     def process_package_artifact(self, filename, processed_filename):
         added_entry = False
         with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
-            for f in JarReader(filename):
-                if not any(mozpath.match(f.filename, p) for p in self.package_artifact_patterns):
+            for p, f in UnpackFinder(JarFinder(filename, JarReader(filename))):
+                if not any(mozpath.match(p, pat) for pat in self.package_artifact_patterns):
                     continue
 
                 # strip off the relative "firefox/" bit from the path:
-                basename = mozpath.relpath(f.filename, "firefox")
+                basename = mozpath.relpath(p, "firefox")
                 basename = mozpath.join('bin', basename)
                 self.log(logging.INFO, 'artifact',
                     {'basename': basename},
                     'Adding {basename} to processed archive')
-                writer.add(basename.encode('utf-8'), f)
+                writer.add(basename.encode('utf-8'), f.open(), mode=f.mode)
                 added_entry = True
 
         if not added_entry:
             raise ValueError('Archive format changed! No pattern from "{patterns}"'
                              'matched an archive path.'.format(
                                  patterns=self.artifact_patterns))
 
 # Keep the keys of this map in sync with the |mach artifact| --job