Bug 1475562 Produce pdbs for the mingw-clang build job r=ted
authorTom Ritter <tom@mozilla.com>
Wed, 17 Oct 2018 09:38:52 -0500
changeset 441846 c06d1f31c0914b09091a7e2d531c782607504d0e
parent 441845 d30c058c4b94021659054523f759a94aee726d3c
child 441847 16ee6006e57cce243c85a5ab7578b43f3a084213
push id34877
push userebalazs@mozilla.com
push dateThu, 18 Oct 2018 10:21:15 +0000
treeherdermozilla-central@8f709fd4aa46 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1475562
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1475562 Produce pdbs for the mingw-clang build job r=ted This patch also changes how pdbs for the ASAN job are copied: we relax restrictions so that pdbs if present) are always copied out and add an environment variable MOZ_COPY_PDBS to indicate when we want to produce pdbs for copying.
browser/config/mozconfigs/win32/mingwclang
browser/config/mozconfigs/win64/mingwclang
build/moz.configure/toolchain.configure
build/win64/mozconfig.asan
config/rules.mk
moz.configure
python/mozbuild/mozbuild/action/check_binary.py
taskcluster/scripts/misc/build-clang-7-mingw.sh
toolkit/mozapps/installer/packager.py
--- a/browser/config/mozconfigs/win32/mingwclang
+++ b/browser/config/mozconfigs/win32/mingwclang
@@ -25,16 +25,17 @@ unset MAKECAB
 #    CARGO
 . "$topsrcdir/browser/config/mozconfigs/common"
 
 # MinGW Stuff
 ac_add_options --target=i686-w64-mingw32
 ac_add_options --with-toolchain-prefix=i686-w64-mingw32-
 
 ac_add_options --disable-warnings-as-errors
+MOZ_COPY_PDBS=1
 
 # Temporary config settings until we get these working on mingw
 ac_add_options --disable-accessibility # https://sourceforge.net/p/mingw-w64/bugs/648/
 
 # For now, we'll disable the sandbox, until we get get Bug 1461421 figured out
 ac_add_options --disable-sandbox
 
 # These aren't supported on mingw at this time
--- a/browser/config/mozconfigs/win64/mingwclang
+++ b/browser/config/mozconfigs/win64/mingwclang
@@ -25,16 +25,17 @@ unset MAKECAB
 #    CARGO
 . "$topsrcdir/browser/config/mozconfigs/common"
 
 # MinGW Stuff
 ac_add_options --target=x86_64-w64-mingw32
 ac_add_options --with-toolchain-prefix=x86_64-w64-mingw32-
 
 ac_add_options --disable-warnings-as-errors
+MOZ_COPY_PDBS=1
 
 # Temporary config settings until we get these working on mingw
 ac_add_options --disable-accessibility # https://sourceforge.net/p/mingw-w64/bugs/648/
 
 # For now, we'll disable the sandbox, until we get get Bug 1461421 figured out
 ac_add_options --disable-sandbox
 
 # These aren't supported on mingw at this time
--- a/build/moz.configure/toolchain.configure
+++ b/build/moz.configure/toolchain.configure
@@ -1121,21 +1121,23 @@ include('compile-checks.configure')
          try_compile(body='static_assert(sizeof(void *) == 8, "")',
                      check_msg='for 64-bit OS'))
 def check_have_64_bit(have_64_bit, compiler_have_64_bit):
     if have_64_bit != compiler_have_64_bit:
         configure_error('The target compiler does not agree with configure '
                         'about the target bitness.')
 
 
-@depends(c_compiler)
-def default_debug_flags(compiler_info):
+@depends(c_compiler, target)
+def default_debug_flags(compiler_info, target):
     # Debug info is ON by default.
     if compiler_info.type in ('msvc', 'clang-cl'):
         return '-Zi'
+    elif target.kernel == 'WINNT' and compiler_info.type == 'clang':
+        return '-g -gcodeview'
     return '-g'
 
 
 option(env='MOZ_DEBUG_FLAGS',
        nargs=1,
        help='Debug compiler flags')
 
 imply_option('--enable-debug-symbols',
@@ -1972,21 +1974,26 @@ add_old_configure_assignment('LIBFUZZER_
 
 # Shared library building
 # ==============================================================
 
 # XXX: The use of makefile constructs in these variables is awful.
 @depends(target, c_compiler)
 def make_shared_library(target, compiler):
     if target.os == 'WINNT':
-        if compiler.type in ('gcc', 'clang'):
+        if compiler.type == 'gcc':
             return namespace(
                 mkshlib=['$(CXX)', '$(DSO_LDOPTS)', '-o', '$@'],
                 mkcshlib=['$(CC)', '$(DSO_LDOPTS)', '-o', '$@'],
             )
+        elif compiler.type == 'clang':
+            return namespace(
+                mkshlib=['$(CXX)', '$(DSO_LDOPTS)', '-Wl,-pdb,$(LINK_PDBFILE)', '-o', '$@'],
+                mkcshlib=['$(CC)', '$(DSO_LDOPTS)', '-Wl,-pdb,$(LINK_PDBFILE)', '-o', '$@'],
+            )
         else:
             linker = [
                 '$(LINKER)',
                 '-NOLOGO', '-DLL',
                 '-OUT:$@',
                 '-PDB:$(LINK_PDBFILE)',
                 '$(DSO_LDOPTS)'
             ]
--- a/build/win64/mozconfig.asan
+++ b/build/win64/mozconfig.asan
@@ -2,16 +2,17 @@
 
 if [ -d "$topsrcdir/clang" ]; then
     CLANG_LIB_DIR="$(cd $topsrcdir/clang/lib/clang/* && cd lib/windows && pwd)"
 
     export LIB=$LIB:$CLANG_LIB_DIR
     mk_export_correct_style LIB
     export LDFLAGS="clang_rt.asan_dynamic-x86_64.lib clang_rt.asan_dynamic_runtime_thunk-x86_64.lib"
 
+    export MOZ_COPY_PDBS=1
     export LLVM_SYMBOLIZER="$topsrcdir/clang/bin/llvm-symbolizer.exe"
     export MOZ_CLANG_RT_ASAN_LIB_PATH="${CLANG_LIB_DIR}/clang_rt.asan_dynamic-x86_64.dll"
 fi
 
 # Enable ASan specific code and build workarounds
 ac_add_options --enable-address-sanitizer
 
 # Mandatory options required for ASan builds
--- a/config/rules.mk
+++ b/config/rules.mk
@@ -121,16 +121,19 @@ MKSHLIB			= $(MKCSHLIB)
 endif
 
 EMBED_MANIFEST_AT=2
 
 endif # MKSHLIB
 endif # FORCE_SHARED_LIB
 
 ifeq ($(OS_ARCH),WINNT)
+
+LINK_PDBFILE ?= $(basename $(@F)).pdb
+
 ifndef GNU_CC
 
 #
 # Unless we're building SIMPLE_PROGRAMS, all C++ files share a PDB file per
 # directory. For parallel builds, this PDB file is shared and locked by
 # MSPDBSRV.EXE, starting with MSVC8 SP1. If you're using MSVC 7.1 or MSVC8
 # without SP1, don't do parallel builds.
 #
@@ -142,30 +145,35 @@ ifndef GNU_CC
 ifdef SIMPLE_PROGRAMS
 COMPILE_PDB_FLAG ?= -Fd$(basename $(@F)).pdb
 else
 COMPILE_PDB_FLAG ?= -Fdgenerated.pdb -FS
 endif
 COMPILE_CFLAGS += $(COMPILE_PDB_FLAG)
 COMPILE_CXXFLAGS += $(COMPILE_PDB_FLAG)
 
-LINK_PDBFILE ?= $(basename $(@F)).pdb
 ifdef MOZ_DEBUG
 CODFILE=$(basename $(@F)).cod
 endif
 
 endif # !GNU_CC
 endif # WINNT
 
 ifeq (arm-Darwin,$(CPU_ARCH)-$(OS_TARGET))
 ifdef PROGRAM
 MOZ_PROGRAM_LDFLAGS += -Wl,-rpath -Wl,@executable_path/Frameworks
 endif
 endif
 
+ifeq ($(OS_ARCH),WINNT)
+ifeq ($(CC_TYPE),clang)
+MOZ_PROGRAM_LDFLAGS += -Wl,-pdb,$(dir $@)/$(LINK_PDBFILE)
+endif
+endif
+
 ifeq ($(HOST_OS_ARCH),WINNT)
 HOST_PDBFILE=$(basename $(@F)).pdb
 HOST_PDB_FLAG ?= -Fd$(HOST_PDBFILE)
 HOST_CFLAGS += $(HOST_PDB_FLAG)
 HOST_CXXFLAGS += $(HOST_PDB_FLAG)
 HOST_C_LDFLAGS += $(HOST_PDB_FLAG)
 HOST_CXX_LDFLAGS += $(HOST_PDB_FLAG)
 endif
@@ -814,23 +822,23 @@ endif
 endif
 
 ifdef MOZ_AUTOMATION
 ifeq (,$(filter 1,$(MOZ_AUTOMATION_BUILD_SYMBOLS)))
 DUMP_SYMS_TARGETS :=
 endif
 endif
 
-ifdef MOZ_CRASHREPORTER
-$(foreach file,$(DUMP_SYMS_TARGETS),$(eval $(call syms_template,$(file),$(notdir $(file))_syms.track)))
-else ifneq (,$(and $(LLVM_SYMBOLIZER),$(filter WINNT,$(OS_ARCH)),$(MOZ_AUTOMATION)))
+ifdef MOZ_COPY_PDBS
 PDB_FILES = $(addsuffix .pdb,$(basename $(DUMP_SYMS_TARGETS)))
 PDB_DEST ?= $(FINAL_TARGET)
 PDB_TARGET = syms
 INSTALL_TARGETS += PDB
+else ifdef MOZ_CRASHREPORTER
+$(foreach file,$(DUMP_SYMS_TARGETS),$(eval $(call syms_template,$(file),$(notdir $(file))_syms.track)))
 endif
 
 cargo_host_flag := --target=$(RUST_HOST_TARGET)
 cargo_target_flag := --target=$(RUST_TARGET)
 
 # Permit users to pass flags to cargo from their mozconfigs (e.g. --color=always).
 cargo_build_flags = $(CARGOFLAGS)
 ifndef MOZ_DEBUG_RUST
--- a/moz.configure
+++ b/moz.configure
@@ -34,16 +34,22 @@ option('--enable-artifact-build-symbols'
 set_config('MOZ_ARTIFACT_BUILD_SYMBOLS',
            depends_if('--enable-artifact-build-symbols')(lambda _: True))
 
 @depends('--enable-artifact-builds')
 def imply_disable_compile_environment(value):
     if value:
         return False
 
+option(env='MOZ_COPY_PDBS',
+    help='For builds that do not support symbols in the normal fashion,'
+         ' generate and copy them into the resulting build archive.')
+
+set_config('MOZ_COPY_PDBS', depends_if('MOZ_COPY_PDBS')(lambda _: True))
+
 imply_option('--enable-compile-environment', imply_disable_compile_environment)
 
 option('--disable-compile-environment',
        help='Disable compiler/library checks')
 
 @depends('--disable-compile-environment')
 def compile_environment(compile_env):
     if compile_env:
--- a/python/mozbuild/mozbuild/action/check_binary.py
+++ b/python/mozbuild/mozbuild/action/check_binary.py
@@ -34,16 +34,17 @@ HOST = {
 
 TARGET = {
     'MOZ_LIBSTDCXX_VERSION':
         buildconfig.substs.get('MOZ_LIBSTDCXX_TARGET_VERSION'),
     'platform': buildconfig.substs['OS_TARGET'],
     'readelf': '{}readelf'.format(
         buildconfig.substs.get('TOOLCHAIN_PREFIX', '')),
     'nm': '{}nm'.format(buildconfig.substs.get('TOOLCHAIN_PREFIX', '')),
+    'readobj': '{}readobj'.format(buildconfig.substs.get('TOOLCHAIN_PREFIX', '')),
 }
 
 if buildconfig.substs.get('HAVE_64BIT_BUILD'):
     GUESSED_NSMODULE_SIZE = 8
 else:
     GUESSED_NSMODULE_SIZE = 4
 
 
@@ -185,17 +186,34 @@ def check_nsmodules(target, binary):
                 name = data[3].split(' = ')[0].split('@@')[0].split('@')[0] \
                               .lstrip('?')
                 if name.endswith('_NSModule') or name in (
                         '__start_kPStaticModules',
                         '__stop_kPStaticModules'):
                     symbols.append((int(data[2], 16), GUESSED_NSMODULE_SIZE,
                                     name))
     else:
-        for line in get_output(target['nm'], '-P', binary):
+        # MinGW-Clang, when building pdbs, doesn't include the symbol table into
+        # the final module. To get the NSModule info, we can look at the exported
+        # symbols. (#1475562)
+        if buildconfig.substs['OS_ARCH'] == 'WINNT' and \
+           buildconfig.substs['HOST_OS_ARCH'] != 'WINNT':
+            readobj_output = get_output(target['readobj'], '-coff-exports', binary)
+            # Transform the output of readobj into nm-like output
+            output = []
+            for line in readobj_output:
+                if "Name" in line:
+                    name = line.replace("Name:", "").strip()
+                elif "RVA" in line:
+                    rva = line.replace("RVA:", "").strip()
+                    output.append("%s r %s" % (name, rva))
+        else:
+            output = get_output(target['nm'], '-P', binary)
+
+        for line in output:
             data = line.split()
             # Some symbols may not have a size listed at all.
             if len(data) == 3:
                 data.append('0')
             if len(data) == 4:
                 sym, _, addr, size = data
                 # NSModules symbols end with _NSModule or _NSModuleE when
                 # C++-mangled.
--- a/taskcluster/scripts/misc/build-clang-7-mingw.sh
+++ b/taskcluster/scripts/misc/build-clang-7-mingw.sh
@@ -270,16 +270,21 @@ build_windres() {
                                                 --disable-multilib \
                                                 --disable-nls \
                                                 --target=$machine-w64-mingw32
   make $make_flags
 
   # Manually install only nm and windres
   cp binutils/windres $INSTALL_DIR/bin/$machine-w64-mingw32-windres
   cp binutils/nm-new $INSTALL_DIR/bin/$machine-w64-mingw32-nm
+
+  pushd $INSTALL_DIR/bin/
+  ln -s llvm-readobj $machine-w64-mingw32-readobj
+  popd
+
   popd
 }
 
 export PATH=$INSTALL_DIR/bin:$PATH
 
 prepare
 
 # gets a bit too verbose here
--- a/toolkit/mozapps/installer/packager.py
+++ b/toolkit/mozapps/installer/packager.py
@@ -306,18 +306,19 @@ def main():
                 libbase = mozpath.join(respath, '%s%s') \
                     % (buildconfig.substs['DLL_PREFIX'], lib)
                 libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
                 if copier.contains(libname):
                     copier.add(libbase + '.chk',
                                LibSignFile(os.path.join(args.destination,
                                                         libname)))
 
-    # Include pdb files for llvm-symbolizer to resolve symbols.
-    if buildconfig.substs.get('LLVM_SYMBOLIZER') and mozinfo.isWin:
+    # If a pdb file is present and we were instructed to copy it, include it.
+    # Run on all OSes to capture MinGW builds
+    if buildconfig.substs.get('MOZ_COPY_PDBS'):
         for p, f in copier:
             if isinstance(f, ExecutableFile):
                 pdbname = os.path.splitext(f.inputs()[0])[0] + '.pdb'
                 if os.path.exists(pdbname):
                     copier.add(os.path.basename(pdbname), File(pdbname))
 
     # Setup preloading
     if args.jarlog and os.path.exists(args.jarlog):