author Ted Mielczarek <>
Fri, 06 Jan 2017 05:57:10 -0500
changeset 328759 e31b6b05ba5b713a6fcac67dbde2eccd591a85dd
parent 324755 d63d083ab7672423620d6d83366d3e1cbae86bad
child 328606 a472aa6efe5ea5541d9d1dbc7d41db8ea0a76bcd
permissions -rw-r--r--
bug 1329320 - replace makecab with rust-makecab in symbolstore. r=gps It turns out that running makecab to compress PDB files takes a significant amount of time in the buildsymbols step. I wrote an implementation of makecab in Rust that implements only the subset of features we use and it's significantly faster: This patch adds a makecab check to moz.configure, adds a release build of the makecab binary to the Windows tooltool manifests, points the build at it from, and changes to use MAKECAB from substs instead of calling `makecab.exe` directly. MozReview-Commit-ID: 76FHLIZFCXS

# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
# vim: set filetype=python:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at

option('--with-windows-version', nargs=1, default='603',
       help='Windows SDK version to target. Win 8.1 (603) is currently'
            'the minimum supported version.')

@imports(_from='__builtin__', _import='ValueError')
def valid_windows_version(value):
    if not value:
        die('Cannot build with --without-windows-version')
        version = int(value[0], 16)
        if version in (0x603,):
            return version
    except ValueError:

    die('Invalid value for --with-windows-version (%s)', value[0])

option(env='WINDOWSSDKDIR', nargs=1,
       help='Directory containing the Windows SDK')

@depends('WINDOWSSDKDIR', host)
def windows_sdk_dir(value, host):
    if value:
        return value
    if host.kernel != 'WINNT':
        return ()

    return tuple(x[1] for x in get_registry_values(
        r'HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows Kits\Installed Roots'

# The Windows SDK 8.1 and 10 have different layouts. The former has
# $SDK/include/$subdir, while the latter has $SDK/include/$version/$subdir.
# The vcvars* scripts don't actually care about the version, they just take
# the last alphanumerically.
# The $SDK/lib directories always have version subdirectories, but while the
# versions match the one in $SDK/include for SDK 10, it's "winv6.3" for SDK
# 8.1.
@imports(_from='__builtin__', _import='sorted')
@imports(_from='__builtin__', _import='WindowsError')
def get_sdk_dirs(sdk, subdir):
    def get_dirs_containing(sdk, stem, subdir):
        base = os.path.join(sdk, stem)
            subdirs = [d for d in os.listdir(base)
                       if os.path.isdir(os.path.join(base, d))]
        except WindowsError:
            subdirs = []
        if not subdirs:
            return ()
        if subdir in subdirs:
            return (base,)
        # At this point, either we have an incomplete or invalid SDK directory,
        # or we exclusively have version numbers in subdirs.
        return tuple(os.path.join(base, s) for s in subdirs
                     if os.path.isdir(os.path.join(base, s, subdir)))

    def categorize(dirs):
        return {os.path.basename(d): d for d in dirs}

    include_dirs = categorize(get_dirs_containing(sdk, 'include', subdir))
    lib_dirs = categorize(get_dirs_containing(sdk, 'lib', subdir))

    if 'include' in include_dirs:
        include_dirs['winv6.3'] = include_dirs['include']
        del include_dirs['include']

    valid_versions = sorted(set(include_dirs) & set(lib_dirs), reverse=True)
    if valid_versions:
        return namespace(

@imports(_from='mozbuild.shellutil', _import='quote')
def valid_windows_sdk_dir_result(value):
    if value:
        return '0x%04x in %s' % (value.version, quote(value.path))

@depends(c_compiler, windows_sdk_dir, valid_windows_version, 'WINDOWSSDKDIR')
@checking('for Windows SDK', valid_windows_sdk_dir_result)
@imports(_from='__builtin__', _import='sorted')
@imports(_from='textwrap', _import='dedent')
def valid_windows_sdk_dir(compiler, windows_sdk_dir, target_version,
    if windows_sdk_dir_env:
        windows_sdk_dir_env = windows_sdk_dir_env[0]
    sdks = {}
    for d in windows_sdk_dir:
        sdk = get_sdk_dirs(d, 'um')
        if sdk:
            check = dedent('''\
            #include <winsdkver.h>
            um_dir = os.path.join(sdk.include, 'um')
            shared_dir = os.path.join(sdk.include, 'shared')
            result = try_preprocess(compiler.wrapper + [compiler.compiler] +
                                    compiler.flags +
                                    ['-I', um_dir, '-I', shared_dir], 'C',
            if result:
                maxver = result.splitlines()[-1]
                    maxver = int(maxver, 0)
                    sdks[d] = maxver, sdk
        if d == windows_sdk_dir_env:
            raise FatalCheckError(
                'Error while checking the version of the SDK in '
                'WINDOWSSDKDIR (%s). Please verify it contains a valid and '
                'complete SDK installation.' % windows_sdk_dir_env)

    valid_sdks = sorted(sdks, key=lambda x: sdks[x][0], reverse=True)
    if valid_sdks:
        biggest_version, sdk = sdks[valid_sdks[0]]
    if not valid_sdks or biggest_version < target_version:
        if windows_sdk_dir_env:
            raise FatalCheckError(
                'You are targeting Windows version 0x%04x, but your SDK only '
                'supports up to version 0x%04x. Install and use an updated SDK, '
                'or target a lower version using --with-windows-version. '
                'Alternatively, try running the Windows SDK Configuration Tool '
                'and selecting a newer SDK. See '
                ' for '
                'details on fixing this.' % (target_version, biggest_version))

        raise FatalCheckError(
            'Cannot find a Windows SDK for version >= 0x%04x.' % target_version)

    return namespace(

    delayed_getattr(valid_windows_sdk_dir, 'path'))
        lambda x: '0x%04X0000' % x.version if x else None))

@imports(_from='mozbuild.shellutil', _import='quote')
def valid_ucrt_sdk_dir_result(value):
    if value:
        return '%s in %s' % (value.version, quote(value.path))

@depends(windows_sdk_dir, 'WINDOWSSDKDIR')
@checking('for Universal CRT SDK', valid_ucrt_sdk_dir_result)
@imports(_from='__builtin__', _import='sorted')
@imports(_import='mozpack.path', _as='mozpath')
def valid_ucrt_sdk_dir(windows_sdk_dir, windows_sdk_dir_env):
    if windows_sdk_dir_env:
        windows_sdk_dir_env = windows_sdk_dir_env[0]
    sdks = {}
    for d in windows_sdk_dir:
        sdk = get_sdk_dirs(d, 'ucrt')
        if sdk:
            version = os.path.basename(sdk.include)
            # We're supposed to always find a version in the directory, because
            # the 8.1 SDK, which doesn't have a version in the directory, doesn't
            # contain the Universal CRT SDK. When the main SDK is 8.1, there
            # is, however, supposed to be a reduced install of the SDK 10
            # with the UCRT.
            if version != 'include':
                sdks[d] = Version(version), sdk
        if d == windows_sdk_dir_env:
            # When WINDOWSSDKDIR is set in the environment and we can't find the
            # Universal CRT SDK, chances are this is a start-shell-msvc*.bat
            # setup, where INCLUDE and LIB already contain the UCRT paths.
            ucrt_includes = [
                p for p in os.environ.get('INCLUDE', '').split(os.pathsep)
                if os.path.basename(p).lower() == 'ucrt'
            ucrt_libs = [
                p for p in os.environ.get('LIB', '').split(os.pathsep)
                if os.path.basename(os.path.dirname(p)).lower() == 'ucrt'
            if ucrt_includes and ucrt_libs:
                # Pick the first of each, since they are the ones that the
                # compiler would look first. Assume they contain the SDK files.
                include = os.path.dirname(ucrt_includes[0])
                lib = os.path.dirname(os.path.dirname(ucrt_libs[0]))
                path = os.path.dirname(os.path.dirname(include))
                version = os.path.basename(include)
                if version != 'include' and mozpath.basedir(lib, [path]):
                    sdks[d] = Version(version), namespace(
            raise FatalCheckError(
                'The SDK in WINDOWSSDKDIR (%s) does not contain the Universal '
                'CRT.' % windows_sdk_dir_env)

    valid_sdks = sorted(sdks, key=lambda x: sdks[x][0], reverse=True)
    if not valid_sdks:
        raise FatalCheckError('Cannot find the Universal CRT SDK. '
                              'Please install it.')

    version, sdk = sdks[valid_sdks[0]]

    return namespace(

def vc_path(c_compiler):
    if c_compiler.type != 'msvc':
    # Normally, we'd start from c_compiler.compiler, but for now, it's not the
    # ideal full path to the compiler. At least, we're guaranteed find_program
    # will get us the one we found in toolchain.configure.
    cl = find_program(c_compiler.compiler)
    result = os.path.dirname(cl)
    while True:
        next, p = os.path.split(result)
        if next == result:
            die('Cannot determine the Visual C++ directory the compiler (%s) '
                'is in' % cl)
        result = next
        if p.lower() == 'bin':
    return result

@checking('for the Debug Interface Access SDK', lambda x: x or 'not found')
@imports(_from='os.path', _import='isdir')
def dia_sdk_dir(vc_path):
    if vc_path:
        path = os.path.join(os.path.dirname(vc_path), 'DIA SDK')
        if isdir(path):
            return path

@depends(vc_path, valid_windows_sdk_dir, valid_ucrt_sdk_dir, dia_sdk_dir)
def include_path(vc_path, windows_sdk_dir, ucrt_sdk_dir, dia_sdk_dir):
    if not vc_path:
    atlmfc_dir = os.path.join(vc_path, 'atlmfc', 'include')
    if not os.path.isdir(atlmfc_dir):
        die('Cannot find the ATL/MFC headers in the Visual C++ directory (%s). '
            'Please install them.' % vc_path)

    winrt_dir = os.path.join(windows_sdk_dir.include, 'winrt')
    if not os.path.isdir(winrt_dir):
        die('Cannot find the WinRT headers in the Windows SDK directory (%s). '
            'Please install them.' % windows_sdk_dir.path)

    includes = []
    include_env = os.environ.get('INCLUDE')
    if include_env:
        os.path.join(vc_path, 'include'),
        os.path.join(windows_sdk_dir.include, 'shared'),
        os.path.join(windows_sdk_dir.include, 'um'),
        os.path.join(ucrt_sdk_dir.include, 'ucrt'),
    if dia_sdk_dir:
        includes.append(os.path.join(dia_sdk_dir, 'include'))
    # Set in the environment for old-configure
    includes = os.pathsep.join(includes)
    os.environ['INCLUDE'] = includes
    return includes

set_config('INCLUDE', include_path)

@depends(target, vc_path, valid_windows_sdk_dir, valid_ucrt_sdk_dir, dia_sdk_dir)
def lib_path(target, vc_path, windows_sdk_dir, ucrt_sdk_dir, dia_sdk_dir):
    if not vc_path:
    vc_target = {
        'x86': '',
        'x86_64': 'amd64',
        'arm': 'arm',
    if vc_target is None:
    # As vc_target can be '', and os.path.join will happily use the empty
    # string, leading to a string ending with a backslash, that Make will
    # interpret as a "string continues on next line" indicator, use variable
    # args.
    vc_target = (vc_target,) if vc_target else ()
    sdk_target = {
        'x86': 'x86',
        'x86_64': 'x64',
        'arm': 'arm',

    atlmfc_dir = os.path.join(vc_path, 'atlmfc', 'lib', *vc_target)
    if not os.path.isdir(atlmfc_dir):
        # For Visual Studio 2017
        atlmfc_dir = os.path.join(vc_path, 'atlmfc', 'lib', sdk_target)
        if not os.path.isdir(atlmfc_dir):
            die('Cannot find the ATL/MFC libraries in the Visual C++ directory '
                '(%s). Please install them.' % vc_path)

    libs = []
    lib_env = os.environ.get('LIB')
    if lib_env:
        os.path.join(vc_path, 'lib', *vc_target),
        os.path.join(windows_sdk_dir.lib, 'um', sdk_target),
        os.path.join(ucrt_sdk_dir.lib, 'ucrt', sdk_target),
    if dia_sdk_dir:
        libs.append(os.path.join(dia_sdk_dir, 'lib', *vc_target))
    # Set in the environment for old-configure
    libs = os.pathsep.join(libs)
    os.environ['LIB'] = libs
    return libs

set_config('LIB', lib_path)

option(env='MT', nargs=1, help='Path to the Microsoft Manifest Tool')

@imports(_from='os', _import='environ')
def sdk_bin_path(valid_windows_sdk_dir):
    if not valid_windows_sdk_dir:

    vc_host = {
        'x86': 'x86',
        'AMD64': 'x64',

    result = [
        os.path.join(valid_windows_sdk_dir.path, 'bin', vc_host)
    if vc_host == 'x64':
            os.path.join(valid_windows_sdk_dir.path, 'bin', 'x86'))
    return result

mt = check_prog('MT', ('mt.exe',), input='MT',

# Check that MT is not something unexpected like "magnetic tape manipulation
# utility".
@checking('whether MT is really Microsoft Manifest Tool', lambda x: bool(x))
def valid_mt(path):
        out = subprocess.check_output([path]).splitlines()
        out = '\n'.join(l for l in out
                        if 'Microsoft (R) Manifest Tool' in l)
        if out:
              return path
    except subprocess.CalledProcessError:
    raise FatalCheckError('%s is not Microsoft Manifest Tool')

set_config('MSMANIFEST_TOOL', depends(valid_mt)(lambda x: bool(x)))

link = check_prog('LINK', ('link.exe',), paths=vc_compiler_path)

add_old_configure_assignment('LINK', link)

# Normally, we'd just have CC, etc. set to absolute paths, but the build system
# doesn't currently handle properly the case where the paths contain spaces.
# Additionally, there's the issue described in toolchain.configure, in
# valid_compiler().
def alter_path(sdk_bin_path):
    path = os.pathsep.join(sdk_bin_path)
    os.environ['PATH'] = path
    return path

set_config('PATH', alter_path)

check_prog('MAKECAB', ('makecab.exe',))