Backed out 6 changesets (bug 1516228) for causing bug 1523056. a=backout
authorCosmin Sabou <csabou@mozilla.com>
Sat, 26 Jan 2019 21:12:33 +0200
changeset 455558 bb2895bfd1bc3d83c309e904dbe74e0c60c3fac9
parent 455554 b08b9f22ad06e55aa83e9c85c74db82c50552094
child 455559 4a038a119dd39fcac171de346ffde7014771f8ec
child 455578 2d2647957a09b9f573042b9b360c7add381b9e3d
push id76854
push usercsabou@mozilla.com
push dateSat, 26 Jan 2019 19:15:06 +0000
treeherderautoland@4a038a119dd3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbackout
bugs1516228, 1523056
milestone66.0a1
backs out24ebb52d21fa602df6aa29c88921c0b8b487bf5c
53d93ee3ad847b03120d3159e2b8b2caace7a370
4f53ede3351741395c132fbbcfb03a06926b7113
2e6bec87c9f38d8621c9fbf5b2099df2c6558ddb
09ee05a4cc30121554f352e9e98d818deb9f8407
8456feb780f7a4b44dc879268e6390aa4f4ce0b1
first release with
nightly linux32
bb2895bfd1bc / 66.0a1 / 20190126191323 / files
nightly linux64
bb2895bfd1bc / 66.0a1 / 20190126191323 / files
nightly mac
bb2895bfd1bc / 66.0a1 / 20190126191323 / files
nightly win32
bb2895bfd1bc / 66.0a1 / 20190126191323 / files
nightly win64
bb2895bfd1bc / 66.0a1 / 20190126191323 / files
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
releases
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out 6 changesets (bug 1516228) for causing bug 1523056. a=backout Backed out changeset 24ebb52d21fa (bug 1516228) Backed out changeset 53d93ee3ad84 (bug 1516228) Backed out changeset 4f53ede33517 (bug 1516228) Backed out changeset 2e6bec87c9f3 (bug 1516228) Backed out changeset 09ee05a4cc30 (bug 1516228) Backed out changeset 8456feb780f7 (bug 1516228)
build/autoconf/compiler-opts.m4
build/moz.configure/init.configure
moz.configure
python/mozbuild/mozbuild/action/check_binary.py
toolkit/library/dependentlibs.py
--- a/build/autoconf/compiler-opts.m4
+++ b/build/autoconf/compiler-opts.m4
@@ -119,17 +119,17 @@ if test "$GNU_CC" -a "$GCC_USE_GNU_LD" -
         LD_SUPPORTS_ICF,
         [echo 'int foo() {return 42;}' \
               'int bar() {return 42;}' \
               'int main() {return foo() - bar();}' > conftest.${ac_ext}
         # If the linker supports ICF, foo and bar symbols will have
         # the same address
         if AC_TRY_COMMAND([${CC-cc} -o conftest${ac_exeext} $LDFLAGS -Wl,--icf=safe -ffunction-sections conftest.${ac_ext} $LIBS 1>&2]) &&
            test -s conftest${ac_exeext} &&
-           $LLVM_OBJDUMP -t conftest${ac_exeext} | awk changequote(<<, >>)'{a[<<$>>6] = <<$>>1} END {if (a["foo"] && (a["foo"] != a["bar"])) { exit 1 }}'changequote([, ]); then
+           objdump -t conftest${ac_exeext} | awk changequote(<<, >>)'{a[<<$>>6] = <<$>>1} END {if (a["foo"] && (a["foo"] != a["bar"])) { exit 1 }}'changequote([, ]); then
             LD_SUPPORTS_ICF=yes
         else
             LD_SUPPORTS_ICF=no
         fi
         rm -rf conftest*])
     if test "$LD_SUPPORTS_ICF" = yes; then
         _SAVE_LDFLAGS="$LDFLAGS -Wl,--icf=safe"
         LDFLAGS="$LDFLAGS -Wl,--icf=safe -Wl,--print-icf-sections"
--- a/build/moz.configure/init.configure
+++ b/build/moz.configure/init.configure
@@ -4,19 +4,16 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 include('util.configure')
 include('checks.configure')
 
 # Make `toolkit` available when toolkit/moz.configure is not included.
 toolkit = dependable(None)
-# Likewise with `bindgen_config_paths` when
-# build/moz.configure/bindgen.configure is not included.
-bindgen_config_paths = dependable(None)
 
 option(env='DIST', nargs=1, help='DIST directory')
 
 
 # Do not allow objdir == srcdir builds.
 # ==============================================================
 @depends('--help', 'DIST')
 @imports(_from='__builtin__', _import='open')
--- a/moz.configure
+++ b/moz.configure
@@ -578,41 +578,16 @@ def nsis_flags(host):
     if host.kernel != 'WINNT':
         return '-nocd'
     return ''
 
 set_config('MAKENSISU_FLAGS', nsis_flags)
 
 check_prog('7Z', ('7z', '7za'), allow_missing=True, when=target_is_windows)
 
-
-@depends(c_compiler, bindgen_config_paths)
-def llvm_objdump(c_compiler, bindgen_config_paths):
-    clang = None
-    if c_compiler and c_compiler.type == 'clang':
-        clang = c_compiler.compiler
-    elif c_compiler and c_compiler.type == 'clang-cl':
-        clang = os.path.join(os.path.dirname(c_compiler.compiler), 'clang')
-    elif bindgen_config_paths:
-        clang = bindgen_config_paths.clang_path
-    llvm_objdump = 'llvm-objdump'
-    if clang:
-        out = check_cmd_output(clang, '--print-prog-name=llvm-objdump',
-                               onerror=lambda: None)
-        if out:
-            llvm_objdump = out.rstrip()
-    return (llvm_objdump,)
-
-
-llvm_objdump = check_prog('LLVM_OBJDUMP', llvm_objdump, what='llvm-objdump',
-                          when='--enable-compile-environment')
-
-add_old_configure_assignment('LLVM_OBJDUMP', llvm_objdump)
-
-
 # Please do not add configure checks from here on.
 
 # Fallthrough to autoconf-based configure
 include('build/moz.configure/old.configure')
 
 # JS Subconfigure.
 include('js/sub.configure', when=compile_environment & toolkit)
 
--- a/python/mozbuild/mozbuild/action/check_binary.py
+++ b/python/mozbuild/mozbuild/action/check_binary.py
@@ -1,53 +1,52 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import print_function, unicode_literals
 
 import argparse
 import os
-import re
 import subprocess
 import sys
 
 from distutils.version import StrictVersion as Version
 
 import buildconfig
 from mozbuild.util import memoize
 from mozpack.executables import (
     get_type,
     ELF,
     MACHO,
-    UNKNOWN,
 )
 
 
 STDCXX_MAX_VERSION = Version('3.4.16')
 GLIBC_MAX_VERSION = Version('2.12')
 LIBGCC_MAX_VERSION = Version('4.8')
 
 HOST = {
     'MOZ_LIBSTDCXX_VERSION':
         buildconfig.substs.get('MOZ_LIBSTDCXX_HOST_VERSION'),
     'platform': buildconfig.substs['HOST_OS_ARCH'],
     'readelf': 'readelf',
+    'nm': 'nm',
 }
 
 TARGET = {
     'MOZ_LIBSTDCXX_VERSION':
         buildconfig.substs.get('MOZ_LIBSTDCXX_TARGET_VERSION'),
     'platform': buildconfig.substs['OS_TARGET'],
     'readelf': '{}readelf'.format(
         buildconfig.substs.get('TOOLCHAIN_PREFIX', '')),
+    'nm': '{}nm'.format(buildconfig.substs.get('TOOLCHAIN_PREFIX', '')),
+    'readobj': '{}readobj'.format(buildconfig.substs.get('TOOLCHAIN_PREFIX', '')),
 }
 
-ADDR_RE = re.compile(r'[0-9a-f]{8,16}')
-
 if buildconfig.substs.get('HAVE_64BIT_BUILD'):
     GUESSED_NSMODULE_SIZE = 8
 else:
     GUESSED_NSMODULE_SIZE = 4
 
 
 get_type = memoize(get_type)
 
@@ -71,107 +70,62 @@ def at_least_one(iter):
     saw_one = False
     for item in iter:
         saw_one = True
         yield item
     if not saw_one:
         raise Empty()
 
 
-# Iterates the symbol table on ELF and MACHO, and the export table on
-# COFF/PE.
-def iter_symbols(binary):
-    ty = get_type(binary)
-    # XXX: Static libraries on ELF, MACHO and COFF/PE systems are all
-    # ar archives. So technically speaking, the following is wrong
-    # but is enough for now. llvm-objdump -t can actually be used on all
-    # platforms for static libraries, but its format is different for
-    # Windows .obj files, so the following won't work for them, but
-    # it currently doesn't matter.
-    if ty == UNKNOWN and open(binary).read(8) == '!<arch>\n':
-        ty = ELF
-    if ty in (ELF, MACHO):
-        for line in get_output(buildconfig.substs['LLVM_OBJDUMP'], '-t',
-                               binary):
-            m = ADDR_RE.match(line)
-            if not m:
-                continue
-            addr = int(m.group(0), 16)
-            # The second "column" is 7 one-character items that can be
-            # whitespaces. We don't have use for their value, so just skip
-            # those.
-            rest = line[m.end() + 9:].split()
-            # The number of remaining colums will vary between ELF and MACHO.
-            # On ELF, we have:
-            #   Section Size .hidden? Name
-            # On Macho, the size is skipped.
-            # In every case, the symbol name is last.
-            name = rest[-1]
+def iter_readelf_symbols(target, binary):
+    for line in get_output(target['readelf'], '-sW', binary):
+        data = line.split()
+        if len(data) >= 8 and data[0].endswith(':') and data[0][:-1].isdigit():
+            n, addr, size, type, bind, vis, index, name = data[:8]
             if '@' in name:
                 name, ver = name.rsplit('@', 1)
                 while name.endswith('@'):
                     name = name[:-1]
             else:
                 ver = None
             yield {
-                'addr': addr,
-                'size': int(rest[1], 16) if ty == ELF else 0,
+                'addr': int(addr, 16),
+                # readelf output may contain decimal values or hexadecimal
+                # values prefixed with 0x for the size. Let python autodetect.
+                'size': int(size, 0),
+                'type': type,
+                'binding': bind,
+                'visibility': vis,
+                'index': index,
                 'name': name,
-                'version': ver or None,
-            }
-    else:
-        export_table = False
-        for line in get_output(buildconfig.substs['LLVM_OBJDUMP'], '-p',
-                               binary):
-            if line.strip() == 'Export Table:':
-                export_table = True
-                continue
-            elif not export_table:
-                continue
-
-            cols = line.split()
-            # The data we're interested in comes in 3 columns, and the first
-            # column is a number.
-            if len(cols) != 3 or not cols[0].isdigit():
-                continue
-            _, rva, name = cols
-            # - The MSVC mangling has some type info following `@@`
-            # - Any namespacing that can happen on the symbol appears as a
-            #   suffix, after a `@`.
-            # - Mangled symbols are prefixed with `?`.
-            name = name.split('@@')[0].split('@')[0].lstrip('?')
-            yield {
-                'addr': int(rva, 16),
-                'size': 0,
-                'name': name,
-                'version': None,
+                'version': ver,
             }
 
 
 def iter_readelf_dynamic(target, binary):
     for line in get_output(target['readelf'], '-d', binary):
         data = line.split(None, 2)
         if data and len(data) == 3 and data[0].startswith('0x'):
             yield data[1].rstrip(')').lstrip('('), data[2]
 
 
 def check_dep_versions(target, binary, lib, prefix, max_version):
     if get_type(binary) != ELF:
         raise Skip()
     unwanted = []
     prefix = prefix + '_'
     try:
-        for sym in at_least_one(iter_symbols(binary)):
-            if sym['addr'] == 0 and sym['version'] and \
+        for sym in at_least_one(iter_readelf_symbols(target, binary)):
+            if sym['index'] == 'UND' and sym['version'] and \
                     sym['version'].startswith(prefix):
                 version = Version(sym['version'][len(prefix):])
                 if version > max_version:
                     unwanted.append(sym)
     except Empty:
-        raise RuntimeError('Could not parse llvm-objdump output?')
+        raise RuntimeError('Could not parse readelf output?')
     if unwanted:
         raise RuntimeError('\n'.join([
             'We do not want these {} symbol versions to be used:'.format(lib)
         ] + [
             ' {} ({})'.format(s['name'], s['version']) for s in unwanted
         ]))
 
 
@@ -213,33 +167,72 @@ def is_libxul(binary):
     basename = os.path.basename(binary).lower()
     return 'xul' in basename
 
 
 def check_nsmodules(target, binary):
     if target is HOST or not is_libxul(binary):
         raise Skip()
     symbols = []
-    for sym in iter_symbols(binary):
-        if sym['addr'] == 0:
-            continue
-        name = sym['name']
-        # NSModules symbols end with _NSModule or _NSModuleE when C++-mangled.
-        if name.endswith(('_NSModule', '_NSModuleE')):
-            # We don't have a valid size in the symbol list for macho and coff.
-            # Use our guesstimate.
-            size = sym['size'] or GUESSED_NSMODULE_SIZE
-            symbols.append((sym['addr'], size, name))
-        elif name in ('__start_kPStaticModules', '__stop_kPStaticModules'):
-            # For coff, these symbols have a size.
-            if get_type(binary) not in (ELF, MACHO):
-                size = GUESSED_NSMODULE_SIZE
-            else:
-                size = 0
-            symbols.append((sym['addr'], size, name))
+    if buildconfig.substs.get('CC_TYPE') in ('msvc', 'clang-cl'):
+        for line in get_output('dumpbin.exe', '-exports', binary):
+            data = line.split(None, 3)
+            if data and len(data) == 4 and data[0].isdigit() and \
+                    ishex(data[1]) and ishex(data[2]):
+                # - Some symbols in the table can be aliases, and appear as
+                #   `foo = bar`.
+                # - The MSVC mangling has some type info following `@@`
+                # - Any namespacing that can happen on the symbol appears as a
+                #   suffix, after a `@`.
+                # - Mangled symbols are prefixed with `?`.
+                name = data[3].split(' = ')[0].split('@@')[0].split('@')[0] \
+                              .lstrip('?')
+                if name.endswith('_NSModule') or name in (
+                        '__start_kPStaticModules',
+                        '__stop_kPStaticModules'):
+                    symbols.append((int(data[2], 16), GUESSED_NSMODULE_SIZE,
+                                    name))
+    else:
+        # MinGW-Clang, when building pdbs, doesn't include the symbol table into
+        # the final module. To get the NSModule info, we can look at the exported
+        # symbols. (#1475562)
+        if buildconfig.substs['OS_ARCH'] == 'WINNT' and \
+           buildconfig.substs['HOST_OS_ARCH'] != 'WINNT':
+            readobj_output = get_output(target['readobj'], '-coff-exports', binary)
+            # Transform the output of readobj into nm-like output
+            output = []
+            for line in readobj_output:
+                if "Name" in line:
+                    name = line.replace("Name:", "").strip()
+                elif "RVA" in line:
+                    rva = line.replace("RVA:", "").strip()
+                    output.append("%s r %s" % (name, rva))
+        else:
+            output = get_output(target['nm'], '-P', binary)
+
+        for line in output:
+            data = line.split()
+            # Some symbols may not have a size listed at all.
+            if len(data) == 3:
+                data.append('0')
+            if len(data) == 4:
+                sym, _, addr, size = data
+                # NSModules symbols end with _NSModule or _NSModuleE when
+                # C++-mangled.
+                if sym.endswith(('_NSModule', '_NSModuleE')):
+                    # On mac, nm doesn't actually print anything other than 0
+                    # for the size. So take our best guess.
+                    size = int(size, 16) or GUESSED_NSMODULE_SIZE
+                    symbols.append((int(addr, 16), size, sym))
+                elif sym.endswith(('__start_kPStaticModules',
+                                   '__stop_kPStaticModules')):
+                    # On ELF and mac systems, these symbols have no size, such
+                    # that the first actual NSModule has the same address as
+                    # the start symbol.
+                    symbols.append((int(addr, 16), 0, sym))
     if not symbols:
         raise RuntimeError('Could not find NSModules')
 
     def print_symbols(symbols):
         for addr, size, sym in symbols:
             print('%x %d %s' % (addr, size, sym))
 
     symbols = sorted(symbols)
--- a/toolkit/library/dependentlibs.py
+++ b/toolkit/library/dependentlibs.py
@@ -14,47 +14,80 @@ import mozpack.path as mozpath
 from collections import OrderedDict
 from mozpack.executables import (
     get_type,
     ELF,
     MACHO,
 )
 from buildconfig import substs
 
-def dependentlibs_win32_objdump(lib):
-    proc = subprocess.Popen([substs['LLVM_OBJDUMP'], '--private-headers', lib], stdout = subprocess.PIPE)
+def dependentlibs_dumpbin(lib):
+    '''Returns the list of dependencies declared in the given DLL'''
+    try:
+        proc = subprocess.Popen(['dumpbin', '-dependents', lib], stdout = subprocess.PIPE)
+    except OSError:
+        # dumpbin is missing, probably mingw compilation. Try using objdump.
+        return dependentlibs_mingw_objdump(lib)
+    deps = []
+    for line in proc.stdout:
+        # Each line containing an imported library name starts with 4 spaces
+        match = re.match('    (\S+)', line)
+        if match:
+             deps.append(match.group(1))
+        elif len(deps):
+             # There may be several groups of library names, but only the
+             # first one is interesting. The second one is for delayload-ed
+             # libraries.
+             break
+    proc.wait()
+    return deps
+
+def dependentlibs_mingw_objdump(lib):
+    try:
+        proc = subprocess.Popen(['objdump', '-x', lib], stdout = subprocess.PIPE)
+    except OSError:
+        # objdump is missing, try using llvm-objdump.
+        proc = subprocess.Popen(['llvm-objdump', '-private-headers', lib], stdout = subprocess.PIPE)
     deps = []
     for line in proc.stdout:
         match = re.match('\s+DLL Name: (\S+)', line)
         if match:
             deps.append(match.group(1))
     proc.wait()
     return deps
 
-def dependentlibs_elf_objdump(lib):
+def dependentlibs_readelf(lib):
     '''Returns the list of dependencies declared in the given ELF .so'''
-    proc = subprocess.Popen([substs['LLVM_OBJDUMP'], '--private-headers', lib], stdout = subprocess.PIPE)
+    proc = subprocess.Popen([substs.get('TOOLCHAIN_PREFIX', '') + 'readelf', '-d', lib], stdout = subprocess.PIPE)
     deps = []
     for line in proc.stdout:
-        # We are looking for lines with the format:
-        #   NEEDED             libname
-        tmp = line.split()
-        if len(tmp) == 2 and tmp[0] == 'NEEDED':
-            deps.append(tmp[1])
+        # Each line has the following format:
+        #  tag (TYPE)          value
+        # or with BSD readelf:
+        #  tag TYPE            value
+        # Looking for NEEDED type entries
+        tmp = line.split(' ', 3)
+        if len(tmp) > 3 and 'NEEDED' in tmp[2]:
+            # NEEDED lines look like:
+            # 0x00000001 (NEEDED)             Shared library: [libname]
+            # or with BSD readelf:
+            # 0x00000001 NEEDED               Shared library: [libname]
+            match = re.search('\[(.*)\]', tmp[3])
+            if match:
+                deps.append(match.group(1))
     proc.wait()
     return deps
 
-def dependentlibs_mac_objdump(lib):
+def dependentlibs_otool(lib):
     '''Returns the list of dependencies declared in the given MACH-O dylib'''
-    proc = subprocess.Popen([substs['LLVM_OBJDUMP'], '--private-headers', lib], stdout = subprocess.PIPE)
-    deps = []
+    proc = subprocess.Popen([substs['OTOOL'], '-l', lib], stdout = subprocess.PIPE)
+    deps= []
     cmd = None
     for line in proc.stdout:
-        # llvm-objdump --private-headers output contains many different
-        # things. The interesting data
+        # otool -l output contains many different things. The interesting data
         # is under "Load command n" sections, with the content:
         #           cmd LC_LOAD_DYLIB
         #       cmdsize 56
         #          name libname (offset 24)
         tmp = line.split()
         if len(tmp) < 2:
             continue
         if tmp[0] == 'cmd':
@@ -85,23 +118,23 @@ def dependentlibs(lib, libpaths, func):
                 break
 
     return deps
 
 def gen_list(output, lib):
     libpaths = [os.path.join(substs['DIST'], 'bin')]
     binary_type = get_type(lib)
     if binary_type == ELF:
-        func = dependentlibs_elf_objdump
+        func = dependentlibs_readelf
     elif binary_type == MACHO:
-        func = dependentlibs_mac_objdump
+        func = dependentlibs_otool
     else:
         ext = os.path.splitext(lib)[1]
         assert(ext == '.dll')
-        func = dependentlibs_win32_objdump
+        func = dependentlibs_dumpbin
 
     deps = dependentlibs(lib, libpaths, func)
     base_lib = mozpath.basename(lib)
     deps[base_lib] = mozpath.join(libpaths[0], base_lib)
     output.write('\n'.join(deps.keys()) + '\n')
 
     with open(output.name + ".gtest", 'w') as gtest_out:
         libs = deps.keys()