Bug 1418464 - Part 1: XZ compress in the packager rather than package_fennec_apk.py. r=mshal
authorNick Alexander <nalexander@mozilla.com>
Tue, 09 Oct 2018 18:35:22 +0000
changeset 496053 a32b2b41d1f7b3e092e486038cc3a6abb84eb80b
parent 496052 f14e8d450327540915e5d28c26a0edb40fcc8a9c
child 496054 bcfb279c5ee80bda24addb09d3110e0b30ee3e17
push id9984
push userffxbld-merge
push dateMon, 15 Oct 2018 21:07:35 +0000
treeherdermozilla-beta@183d27ea8570 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmshal
bugs1418464
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1418464 - Part 1: XZ compress in the packager rather than package_fennec_apk.py. r=mshal By doing this in the packager, it makes it easier to incorporate the strip and XZ compress logic into the local Gradle build process. To that end, this patch makes XZ compression a little more explicit in package-manifest.in and lifts the logic next to the existing logic for stripping. Since we only want to XZ compress assets/ (and not libs/), we need a new flag. Differential Revision: https://phabricator.services.mozilla.com/D7314
mobile/android/installer/package-manifest.in
python/mozbuild/mozbuild/action/package_fennec_apk.py
python/mozbuild/mozpack/executables.py
python/mozbuild/mozpack/files.py
python/mozbuild/mozpack/packager/__init__.py
--- a/mobile/android/installer/package-manifest.in
+++ b/mobile/android/installer/package-manifest.in
@@ -19,17 +19,17 @@
 @BINPATH@/update.locale
 #ifdef MOZ_UPDATER
 @BINPATH@/updater.ini
 #endif
 @BINPATH@/dictionaries/*
 @BINPATH@/hyphenation/*
 @BINPATH@/localization/*
 
-[assets destdir="assets/@ANDROID_CPU_ARCH@"]
+[assets xz_compress="1" destdir="assets/@ANDROID_CPU_ARCH@"]
 #ifndef MOZ_STATIC_JS
 @BINPATH@/@DLL_PREFIX@mozjs@DLL_SUFFIX@
 #endif
 #ifdef MOZ_DMD
 @BINPATH@/@DLL_PREFIX@dmd@DLL_SUFFIX@
 #endif
 #ifndef MOZ_FOLD_LIBS
 @BINPATH@/@DLL_PREFIX@plc4@DLL_SUFFIX@
--- a/python/mozbuild/mozbuild/action/package_fennec_apk.py
+++ b/python/mozbuild/mozbuild/action/package_fennec_apk.py
@@ -87,47 +87,16 @@ def package_fennec_apk(inputs=[], omni_j
             compress = None  # Take default from Jarrer.
             if p.endswith('.so'):
                 # Asset libraries are special.
                 if f.open().read(5)[1:] == '7zXZ':
                     print('%s is already compressed' % p)
                     # We need to store (rather than deflate) compressed libraries
                     # (even if we don't compress them ourselves).
                     compress = False
-                elif buildconfig.substs.get('XZ'):
-                    cmd = [buildconfig.substs.get('XZ'), '-zkf',
-                           mozpath.join(finder.base, p)]
-
-                    # For now, the mozglue XZStream ELF loader can only support xz files
-                    # with a single stream that contains a single block. In xz, there is no
-                    # explicit option to set the max block count. Instead, we force xz to use
-                    # single thread mode, which results in a single block.
-                    cmd.extend(['--threads=1'])
-
-                    bcj = None
-                    if buildconfig.substs.get('MOZ_THUMB2'):
-                        bcj = '--armthumb'
-                    elif buildconfig.substs.get('CPU_ARCH') == 'arm':
-                        bcj = '--arm'
-                    elif buildconfig.substs.get('CPU_ARCH') == 'x86':
-                        bcj = '--x86'
-
-                    if bcj:
-                        cmd.extend([bcj])
-                    # We need to explicitly specify the LZMA filter chain to ensure consistent builds
-                    # across platforms. Note that the dict size must be less then 16MiB per the hardcoded
-                    # value in mozglue/linker/XZStream.cpp. This is the default LZMA filter chain for for
-                    # xz-utils version 5.0. See:
-                    # https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/lzma/lzma_encoder_presets.c
-                    # https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/api/lzma/container.h#L31
-                    cmd.extend(['--lzma2=dict=8MiB,lc=3,lp=0,pb=2,mode=normal,nice=64,mf=bt4,depth=0'])
-                    print('xz-compressing %s with %s' % (p, ' '.join(cmd)))
-                    subprocess.check_output(cmd)
-                    os.rename(f.path + '.xz', f.path)
-                    compress = False
 
             add(mozpath.join('assets', p), f, compress=compress)
 
     for lib_dir in lib_dirs:
         finder = FileFinder(lib_dir)
         for p, f in finder.find('**'):
             add(mozpath.join('lib', p), f)
 
--- a/python/mozbuild/mozpack/executables.py
+++ b/python/mozbuild/mozpack/executables.py
@@ -117,8 +117,53 @@ def elfhack(path):
     Execute the elfhack command on the given path.
     '''
     from buildconfig import topobjdir
     cmd = [os.path.join(topobjdir, 'build/unix/elfhack/elfhack'), path]
     if 'ELF_HACK_FLAGS' in os.environ:
         cmd[1:0] = os.environ['ELF_HACK_FLAGS'].split()
     if subprocess.call(cmd) != 0:
         errors.fatal('Error executing ' + ' '.join(cmd))
+
+
+def xz_compress(path):
+    '''
+    Execute xz to compress the given path.
+    '''
+    if open(path, 'rb').read(5)[1:] == '7zXZ':
+        print('%s is already compressed' % path)
+        return
+    
+    from buildconfig import substs
+    xz = substs.get('XZ')
+    cmd = [xz, '-zkf', path]
+
+    # For now, the mozglue XZStream ELF loader can only support xz files
+    # with a single stream that contains a single block. In xz, there is no
+    # explicit option to set the max block count. Instead, we force xz to use
+    # single thread mode, which results in a single block.
+    cmd.extend(['--threads=1'])
+
+    bcj = None
+    if substs.get('MOZ_THUMB2'):
+        bcj = '--armthumb'
+    elif substs.get('CPU_ARCH') == 'arm':
+        bcj = '--arm'
+    elif substs.get('CPU_ARCH') == 'x86':
+        bcj = '--x86'
+
+    if bcj:
+        cmd.extend([bcj])
+
+    # We need to explicitly specify the LZMA filter chain to ensure consistent builds
+    # across platforms. Note that the dict size must be less then 16MiB per the hardcoded
+    # value in mozglue/linker/XZStream.cpp. This is the default LZMA filter chain for for
+    # xz-utils version 5.0. See:
+    # https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/lzma/lzma_encoder_presets.c
+    # https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/api/lzma/container.h#L31
+    cmd.extend(['--lzma2=dict=8MiB,lc=3,lp=0,pb=2,mode=normal,nice=64,mf=bt4,depth=0'])
+    print('xz-compressing %s with %s' % (path, ' '.join(cmd)))
+
+    if subprocess.call(cmd) != 0:
+        errors.fatal('Error executing ' + ' '.join(cmd))
+        return
+
+    os.rename(path + '.xz', path)
--- a/python/mozbuild/mozpack/files.py
+++ b/python/mozbuild/mozpack/files.py
@@ -16,16 +16,17 @@ from itertools import chain
 from mozbuild.preprocessor import Preprocessor
 from mozbuild.util import FileAvoidWrite
 from mozpack.executables import (
     is_executable,
     may_strip,
     strip,
     may_elfhack,
     elfhack,
+    xz_compress,
 )
 from mozpack.chrome.manifest import (
     ManifestEntry,
     ManifestInterfaces,
 )
 from io import BytesIO
 from mozpack.errors import (
     ErrorMessage,
@@ -275,34 +276,40 @@ class File(BaseFile):
         return (self.path,)
 
 
 class ExecutableFile(File):
     '''
     File class for executable and library files on OS/2, OS/X and ELF systems.
     (see mozpack.executables.is_executable documentation).
     '''
+    def __init__(self, path, xz_compress=False):
+        File.__init__(self, path)
+        self.xz_compress = xz_compress
+
     def copy(self, dest, skip_if_older=True):
         real_dest = dest
         if not isinstance(dest, basestring):
             fd, dest = mkstemp()
             os.close(fd)
             os.remove(dest)
         assert isinstance(dest, basestring)
         # If File.copy didn't actually copy because dest is newer, check the
         # file sizes. If dest is smaller, it means it is already stripped and
-        # elfhacked, so we can skip.
+        # elfhacked and xz_compressed, so we can skip.
         if not File.copy(self, dest, skip_if_older) and \
                 os.path.getsize(self.path) > os.path.getsize(dest):
             return False
         try:
             if may_strip(dest):
                 strip(dest)
             if may_elfhack(dest):
                 elfhack(dest)
+            if self.xz_compress:
+                xz_compress(dest)
         except ErrorMessage:
             os.remove(dest)
             raise
 
         if real_dest != dest:
             f = File(dest)
             ret = f.copy(real_dest, skip_if_older)
             os.remove(dest)
--- a/python/mozbuild/mozpack/packager/__init__.py
+++ b/python/mozbuild/mozpack/packager/__init__.py
@@ -11,46 +11,56 @@ from mozpack.errors import errors
 from mozpack.chrome.manifest import (
     Manifest,
     ManifestBinaryComponent,
     ManifestChrome,
     ManifestInterfaces,
     is_manifest,
     parse_manifest,
 )
+from mozpack.files import (
+    ExecutableFile,
+)
 import mozpack.path as mozpath
 from collections import deque
 import json
 
 
 class Component(object):
     '''
     Class that represents a component in a package manifest.
     '''
-    def __init__(self, name, destdir=''):
+    def __init__(self, name, destdir='', xz_compress=False):
         if name.find(' ') > 0:
             errors.fatal('Malformed manifest: space in component name "%s"'
                          % component)
         self._name = name
         self._destdir = destdir
+        self._xz_compress = xz_compress
 
     def __repr__(self):
         s = self.name
         if self.destdir:
             s += ' destdir="%s"' % self.destdir
+        if self.xz_compress:
+            s += ' xz_compress="1"'
         return s
 
     @property
     def name(self):
         return self._name
 
     @property
     def destdir(self):
         return self._destdir
 
+    @property
+    def xz_compress(self):
+        return self._xz_compress
+
     @staticmethod
     def _triples(lst):
         '''
         Split [1, 2, 3, 4, 5, 6, 7] into [(1, 2, 3), (4, 5, 6)].
         '''
         return zip(*[iter(lst)] * 3)
 
     KEY_VALUE_RE = re.compile(r'''
@@ -112,20 +122,21 @@ class Component(object):
         Create a component from a string.
         '''
         try:
             name, options = Component._split_component_and_options(string)
         except ValueError as e:
             errors.fatal('Malformed manifest: %s' % e)
             return
         destdir = options.pop('destdir', '')
+        xz_compress = options.pop('xz_compress', '0') != '0'
         if options:
             errors.fatal('Malformed manifest: options %s not recognized'
                          % options.keys())
-        return Component(name, destdir=destdir)
+        return Component(name, destdir=destdir, xz_compress=xz_compress)
 
 
 class PackageManifestParser(object):
     '''
     Class for parsing of a package manifest, after preprocessing.
 
     A package manifest is a list of file paths, with some syntaxic sugar:
         [] designates a toplevel component. Example: [xpcom]
@@ -400,16 +411,18 @@ class SimpleManifestSink(object):
         '''
         assert not self._closed
         added = False
         for p, f in self._finder.find(pattern):
             added = True
             if is_manifest(p):
                 self._manifests.add(p)
             dest = mozpath.join(component.destdir, SimpleManifestSink.normalize_path(p))
+            if isinstance(f, ExecutableFile):
+                f.xz_compress = component.xz_compress
             self.packager.add(dest, f)
         if not added:
             errors.error('Missing file(s): %s' % pattern)
 
     def remove(self, component, pattern):
         '''
         Remove files with the given pattern in the given component.
         '''