toolkit/mozapps/installer/packager.py
author Mike Hommey <mh+mozilla@glandium.org>
Wed, 06 Mar 2019 01:18:10 +0000
changeset 462549 3c37fd9d30d50b66b9b899c3f85758079f8e31d1
parent 461391 2bb574d4377e1d7f40d247519cac8fc586aecf19
child 462576 24e79b0a187fdff462d085bc50b1adcc269e3509
permissions -rw-r--r--
Bug 1529894 - Change jar log content. r=aklotz,chmanchester The jar log is used for optimization of the packaged jar files according to their usage patterns during a profile run. The current content of the file currently come with 2 caveats: - it contains entries for jar archives that aren't relevant to packaging, which is not a problem in itself, but see below. - it contains full paths for jar archives that may not correspond to the location of the packaged directory (on e.g. Android, where the build almost certainly doesn't happen in the same directory on the host as Fennec runs in the emulator/on the device). The current JarLog code does somehow handle the various ways paths are currently presented, but it's clearly missing code to map the paths in the log to packaged paths. Instead of requiring manual work and extra build options to handle this mapping, and considering the caveats above, it's just simpler to log archive paths as if they were relative to the packaged application directory in a build, and use that during packaging. Depends on D21655 Differential Revision: https://phabricator.services.mozilla.com/D21656

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

from mozpack.packager.formats import (
    FlatFormatter,
    JarFormatter,
    OmniJarFormatter,
)
from mozpack.packager import (
    preprocess_manifest,
    preprocess,
    Component,
    SimpleManifestSink,
)
from mozpack.files import (
    GeneratedFile,
    FileFinder,
    File,
)
from mozpack.copier import (
    FileCopier,
    Jarrer,
)
from mozpack.errors import errors
from mozpack.files import ExecutableFile
from mozpack.mozjar import JAR_BROTLI
import mozpack.path as mozpath
import buildconfig
from argparse import ArgumentParser
from createprecomplete import generate_precomplete
import os
from StringIO import StringIO
import subprocess
import mozinfo

# List of libraries to shlibsign.
SIGN_LIBS = [
    'softokn3',
    'nssdbm3',
    'freebl3',
    'freeblpriv3',
    'freebl_32fpu_3',
    'freebl_32int_3',
    'freebl_32int64_3',
    'freebl_64fpu_3',
    'freebl_64int_3',
]


class ToolLauncher(object):
    '''
    Helper to execute tools like xpcshell with the appropriate environment.
        launcher = ToolLauncher()
        launcher.tooldir = '/path/to/tools'
        launcher.launch(['xpcshell', '-e', 'foo.js'])
    '''
    def __init__(self):
        self.tooldir = None

    def launch(self, cmd, extra_linker_path=None, extra_env={}):
        '''
        Launch the given command, passed as a list. The first item in the
        command list is the program name, without a path and without a suffix.
        These are determined from the tooldir member and the BIN_SUFFIX value.
        An extra_linker_path may be passed to give an additional directory
        to add to the search paths for the dynamic linker.
        An extra_env dict may be passed to give additional environment
        variables to export when running the command.
        '''
        assert self.tooldir
        cmd[0] = os.path.join(self.tooldir, 'bin',
                              cmd[0] + buildconfig.substs['BIN_SUFFIX'])
        if not extra_linker_path:
            extra_linker_path = os.path.join(self.tooldir, 'bin')
        env = dict(os.environ)
        for p in ['LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH']:
            if p in env:
                env[p] = extra_linker_path + ':' + env[p]
            else:
                env[p] = extra_linker_path
        for e in extra_env:
            env[e] = extra_env[e]

        # For VC12+, make sure we can find the right bitness of pgort1x0.dll
        if not buildconfig.substs.get('HAVE_64BIT_BUILD'):
            for e in ('VS140COMNTOOLS', 'VS120COMNTOOLS'):
                if e not in env:
                    continue

                vcdir = os.path.abspath(os.path.join(env[e], '../../VC/bin'))
                if os.path.exists(vcdir):
                    env['PATH'] = '%s;%s' % (vcdir, env['PATH'])
                    break

        # Work around a bug in Python 2.7.2 and lower where unicode types in
        # environment variables aren't handled by subprocess.
        for k, v in env.items():
            if isinstance(v, unicode):
                env[k] = v.encode('utf-8')

        print >>errors.out, 'Executing', ' '.join(cmd)
        errors.out.flush()
        return subprocess.call(cmd, env=env)

    def can_launch(self):
        return self.tooldir is not None

launcher = ToolLauncher()


class LibSignFile(File):
    '''
    File class for shlibsign signatures.
    '''
    def copy(self, dest, skip_if_older=True):
        assert isinstance(dest, basestring)
        # os.path.getmtime returns a result in seconds with precision up to the
        # microsecond. But microsecond is too precise because shutil.copystat
        # only copies milliseconds, and seconds is not enough precision.
        if os.path.exists(dest) and skip_if_older and \
                int(os.path.getmtime(self.path) * 1000) <= \
                int(os.path.getmtime(dest) * 1000):
            return False
        if launcher.launch(['shlibsign', '-v', '-o', dest, '-i', self.path]):
            errors.fatal('Error while signing %s' % self.path)


class RemovedFiles(GeneratedFile):
    '''
    File class for removed-files. Is used as a preprocessor parser.
    '''
    def __init__(self, copier):
        self.copier = copier
        GeneratedFile.__init__(self, '')

    def handle_line(self, str):
        f = str.strip()
        if not f:
            return
        if self.copier.contains(f):
            errors.error('Removal of packaged file(s): %s' % f)
        self.content += f + '\n'


def split_define(define):
    '''
    Give a VAR[=VAL] string, returns a (VAR, VAL) tuple, where VAL defaults to
    1. Numeric VALs are returned as ints.
    '''
    if '=' in define:
        name, value = define.split('=', 1)
        try:
            value = int(value)
        except ValueError:
            pass
        return (name, value)
    return (define, 1)


class NoPkgFilesRemover(object):
    '''
    Formatter wrapper to handle NO_PKG_FILES.
    '''
    def __init__(self, formatter, has_manifest):
        assert 'NO_PKG_FILES' in os.environ
        self._formatter = formatter
        self._files = os.environ['NO_PKG_FILES'].split()
        if has_manifest:
            self._error = errors.error
            self._msg = 'NO_PKG_FILES contains file listed in manifest: %s'
        else:
            self._error = errors.warn
            self._msg = 'Skipping %s'

    def add_base(self, base, *args):
        self._formatter.add_base(base, *args)

    def add(self, path, content):
        if not any(mozpath.match(path, spec) for spec in self._files):
            self._formatter.add(path, content)
        else:
            self._error(self._msg % path)

    def add_manifest(self, entry):
        self._formatter.add_manifest(entry)

    def contains(self, path):
        return self._formatter.contains(path)


def main():
    parser = ArgumentParser()
    parser.add_argument('-D', dest='defines', action='append',
                        metavar="VAR[=VAL]", help='Define a variable')
    parser.add_argument('--format', default='omni',
                        help='Choose the chrome format for packaging ' +
                        '(omni, jar or flat ; default: %(default)s)')
    parser.add_argument('--removals', default=None,
                        help='removed-files source file')
    parser.add_argument('--ignore-errors', action='store_true', default=False,
                        help='Transform errors into warnings.')
    parser.add_argument('--minify', action='store_true', default=False,
                        help='Make some files more compact while packaging')
    parser.add_argument('--minify-js', action='store_true',
                        help='Minify JavaScript files while packaging.')
    parser.add_argument('--js-binary',
                        help='Path to js binary. This is used to verify '
                        'minified JavaScript. If this is not defined, '
                        'minification verification will not be performed.')
    parser.add_argument('--jarlog', default='', help='File containing jar ' +
                        'access logs')
    parser.add_argument('--compress', choices=('none', 'deflate', 'brotli'),
                        default='deflate',
                        help='Use given jar compression (default: deflate)')
    parser.add_argument('manifest', default=None, nargs='?',
                        help='Manifest file name')
    parser.add_argument('source', help='Source directory')
    parser.add_argument('destination', help='Destination directory')
    parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
                        default=[],
                        help='Extra files not to be considered as resources')
    args = parser.parse_args()

    defines = dict(buildconfig.defines['ALLDEFINES'])
    if args.ignore_errors:
        errors.ignore_errors()

    if args.defines:
        for name, value in [split_define(d) for d in args.defines]:
            defines[name] = value

    compress = {
        'none': False,
        'deflate': True,
        'brotli': JAR_BROTLI,
    }[args.compress]

    copier = FileCopier()
    if args.format == 'flat':
        formatter = FlatFormatter(copier)
    elif args.format == 'jar':
        formatter = JarFormatter(copier, compress=compress)
    elif args.format == 'omni':
        formatter = OmniJarFormatter(copier,
                                     buildconfig.substs['OMNIJAR_NAME'],
                                     compress=compress,
                                     non_resources=args.non_resource)
    else:
        errors.fatal('Unknown format: %s' % args.format)

    # Adjust defines according to the requested format.
    if isinstance(formatter, OmniJarFormatter):
        defines['MOZ_OMNIJAR'] = 1
    elif 'MOZ_OMNIJAR' in defines:
        del defines['MOZ_OMNIJAR']

    respath = ''
    if 'RESPATH' in defines:
        respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
    while respath.startswith('/'):
        respath = respath[1:]

    if not buildconfig.substs['CROSS_COMPILE']:
        launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist')

    with errors.accumulate():
        finder_args = dict(
            minify=args.minify,
            minify_js=args.minify_js,
        )
        if args.js_binary:
            finder_args['minify_js_verify_command'] = [
                args.js_binary,
                os.path.join(os.path.abspath(os.path.dirname(__file__)),
                    'js-compare-ast.js')
            ]
        finder = FileFinder(args.source, find_executables=True,
                            **finder_args)
        if 'NO_PKG_FILES' in os.environ:
            sinkformatter = NoPkgFilesRemover(formatter,
                                              args.manifest is not None)
        else:
            sinkformatter = formatter
        sink = SimpleManifestSink(finder, sinkformatter)
        if args.manifest:
            preprocess_manifest(sink, args.manifest, defines)
        else:
            sink.add(Component(''), 'bin/*')
        sink.close(args.manifest is not None)

        if args.removals:
            removals_in = StringIO(open(args.removals).read())
            removals_in.name = args.removals
            removals = RemovedFiles(copier)
            preprocess(removals_in, removals, defines)
            copier.add(mozpath.join(respath, 'removed-files'), removals)

    # shlibsign libraries
    if launcher.can_launch():
        if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'):
            for lib in SIGN_LIBS:
                libbase = mozpath.join(respath, '%s%s') \
                    % (buildconfig.substs['DLL_PREFIX'], lib)
                libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
                if copier.contains(libname):
                    copier.add(libbase + '.chk',
                               LibSignFile(os.path.join(args.destination,
                                                        libname)))

    # If a pdb file is present and we were instructed to copy it, include it.
    # Run on all OSes to capture MinGW builds
    if buildconfig.substs.get('MOZ_COPY_PDBS'):
        for p, f in copier:
            if isinstance(f, ExecutableFile):
                pdbname = os.path.splitext(f.inputs()[0])[0] + '.pdb'
                if os.path.exists(pdbname):
                    copier.add(os.path.basename(pdbname), File(pdbname))

    # Setup preloading
    if args.jarlog and os.path.exists(args.jarlog):
        from mozpack.mozjar import JarLog
        log = JarLog(args.jarlog)
        for p, f in copier:
            if not isinstance(f, Jarrer):
                continue
            if p in log:
                f.preload(log[p])

    copier.copy(args.destination)
    generate_precomplete(os.path.normpath(os.path.join(args.destination,
                                                       respath)))


if __name__ == '__main__':
    main()