Bug 1469088 - Relax the assumptions of --enable-lto, and make it work for macOS builds. r=ted
authorMike Hommey <mh+mozilla@glandium.org>
Thu, 05 Jul 2018 11:35:31 +0900 (2018-07-05)
changeset 425412 e8c173a632a43ca26f2e0d7eb4ca362ece212ef4
parent 425411 44861ed17925f75e8b2ef0109b5b0c655cccc95d
child 425413 79ef83e82742de2911844ab13315072dc2306a74
push id34245
push userdluca@mozilla.com
push dateSat, 07 Jul 2018 10:29:39 +0000 (2018-07-07)
treeherdermozilla-central@1df83f663c81 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs1469088
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1469088 - Relax the assumptions of --enable-lto, and make it work for macOS builds. r=ted Currently, --enable-lto just implies --enable-linker=lld, which essentially only works on Linux, and assumes one can't do lto with anything other than lld. Which is not true. As a matter of fact, even ld.bfd can do lto, as long as the gold plugin for llvm is available, or when doing lto with GCC instead of clang. Anyways, to allow more lto setups, we adapt the --enable-linker setup to: - work on macOS, which it currently doesn't, and add support for the mac linker (ld64), which, unfortunately, doesn't have a clean way to be detected, so work around that. - default to lld if lto is enable, no linker was explicitly given, the compiler is clang *and* the build target is not macOS.
build/moz.configure/toolchain.configure
build/moz.configure/util.configure
--- a/build/moz.configure/toolchain.configure
+++ b/build/moz.configure/toolchain.configure
@@ -1353,18 +1353,16 @@ def lto(value, c_compiler):
         ldflags=ldflags
     )
 
 
 add_old_configure_assignment('MOZ_LTO', lto.enabled)
 add_old_configure_assignment('MOZ_LTO_CFLAGS', lto.flags)
 add_old_configure_assignment('MOZ_LTO_LDFLAGS', lto.ldflags)
 
-imply_option('--enable-linker', 'lld', when='--enable-lto')
-
 # ASAN
 # ==============================================================
 
 js_option('--enable-address-sanitizer', help='Enable Address Sanitizer')
 
 
 @depends_if('--enable-address-sanitizer')
 def asan(value):
@@ -1557,63 +1555,99 @@ def cargo_incremental(opt_level, debug_r
 set_config('CARGO_INCREMENTAL', cargo_incremental)
 
 # Linker detection
 # ==============================================================
 
 
 @depends(target)
 def is_linker_option_enabled(target):
-    if target.kernel not in ('Darwin', 'WINNT', 'SunOS'):
+    if target.kernel not in ('WINNT', 'SunOS'):
         return True
 
 
 option('--enable-gold',
        env='MOZ_FORCE_GOLD',
        help='Enable GNU Gold Linker when it is not already the default',
        when=is_linker_option_enabled)
 
 imply_option('--enable-linker', 'gold', when='--enable-gold')
 
 js_option('--enable-linker', nargs=1,
-          help='Select the linker {bfd, gold, lld, lld-*}',
+          help='Select the linker {bfd, gold, ld64, lld, lld-*}',
           when=is_linker_option_enabled)
 
 
 @depends('--enable-linker', c_compiler, developer_options, '--enable-gold',
-         extra_toolchain_flags, when=is_linker_option_enabled)
+         extra_toolchain_flags, target, lto.enabled,
+         when=is_linker_option_enabled)
 @checking('for linker', lambda x: x.KIND)
 @imports('os')
 @imports('shutil')
 def select_linker(linker, c_compiler, developer_options, enable_gold,
-                  toolchain_flags):
+                  toolchain_flags, target, lto):
+
+    if linker:
+        linker = linker[0]
+    elif lto and c_compiler.type == 'clang' and target.kernel != 'Darwin':
+        # If no linker was explicitly given, and building with clang for non-macOS,
+        # prefer lld. For macOS, we prefer ld64, or whatever the default linker is.
+        linker = 'lld'
+    else:
+        linker = None
 
-    linker = linker[0] if linker else None
+    def is_valid_linker(linker):
+        if target.kernel == 'Darwin':
+            valid_linkers = ('ld64', 'lld')
+        else:
+            valid_linkers = ('bfd', 'gold', 'lld')
+        if linker in valid_linkers:
+            return True
+        if 'lld' in valid_linkers and linker.startswith('lld-'):
+            return True
+        return False
 
-    if linker not in ('bfd', 'gold', 'lld', None) and not linker.startswith("lld-"):
+    if linker and not is_valid_linker(linker):
         # Check that we are trying to use a supported linker
         die('Unsupported linker ' + linker)
 
     # Check the kind of linker
     version_check = ['-Wl,--version']
     cmd_base = c_compiler.wrapper + [c_compiler.compiler] + c_compiler.flags
 
     def try_linker(linker):
         # Generate the compiler flag
-        linker_flag = ["-fuse-ld=" + linker] if linker else []
+        if linker == 'ld64':
+            linker_flag = ['-fuse-ld=ld']
+        elif linker:
+            linker_flag = ["-fuse-ld=" + linker]
+        else:
+            linker_flag = []
         cmd = cmd_base + linker_flag + version_check
         if toolchain_flags:
             cmd += toolchain_flags
 
-        cmd_output = check_cmd_output(*cmd, onerror=lambda: None)
-        if cmd_output is None:
+        # ld64 doesn't have anything to print out a version. It does print out
+        # "ld64: For information on command line options please use 'man ld'."
+        # but that would require doing two attempts, one with --version, that
+        # would fail, and another with --help.
+        # Instead, abuse its LD_PRINT_OPTIONS feature to detect a message
+        # specific to it on stderr when it fails to process --version.
+        env = dict(os.environ)
+        env['LD_PRINT_OPTIONS'] = '1'
+        retcode, stdout, stderr = get_cmd_output(*cmd, env=env)
+        cmd_output = stdout.decode('utf-8')
+        stderr = stderr.decode('utf-8')
+        if retcode == 1 and 'Logging ld64 options' in stderr:
+            kind = 'ld64'
+
+        elif retcode != 0:
             return None
-        cmd_output = cmd_output.decode('utf-8')
 
-        if 'GNU ld' in cmd_output:
+        elif 'GNU ld' in cmd_output:
             # We are using the normal linker
             kind = 'bfd'
 
         elif 'GNU gold' in cmd_output:
             kind = 'gold'
 
         elif 'LLD' in cmd_output:
             kind = 'lld'
--- a/build/moz.configure/util.configure
+++ b/build/moz.configure/util.configure
@@ -14,52 +14,57 @@ def die(*args):
 
 @imports(_from='mozbuild.configure', _import='ConfigureError')
 def configure_error(message):
     '''Raise a programming error and terminate configure.
     Primarily for use in moz.configure templates to sanity check
     their inputs from moz.configure usage.'''
     raise ConfigureError(message)
 
-# A wrapper to obtain a process' output that returns the output generated
-# by running the given command if it exits normally, and streams that
-# output to log.debug and calls die or the given error callback if it
-# does not.
 
-
+# A wrapper to obtain a process' output and return code.
+# Returns a tuple (retcode, stdout, stderr).
 @imports(_from='__builtin__', _import='unicode')
 @imports('subprocess')
-@imports('sys')
-@imports(_from='mozbuild.configure.util', _import='LineIO')
 @imports(_from='mozbuild.shellutil', _import='quote')
-def check_cmd_output(*args, **kwargs):
-    onerror = kwargs.pop('onerror', None)
-
+def get_cmd_output(*args, **kwargs):
     # subprocess on older Pythons can't handle unicode keys or values in
     # environment dicts. Normalize automagically so callers don't have to
     # deal with this.
     if 'env' in kwargs:
         normalized_env = {}
         for k, v in kwargs['env'].items():
             if isinstance(k, unicode):
                 k = k.encode('utf-8', 'strict')
 
             if isinstance(v, unicode):
                 v = v.encode('utf-8', 'strict')
 
             normalized_env[k] = v
 
         kwargs['env'] = normalized_env
 
+    log.debug('Executing: `%s`', quote(*args))
+    proc = subprocess.Popen(args, stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE, **kwargs)
+    stdout, stderr = proc.communicate()
+    return proc.wait(), stdout, stderr
+
+
+# A wrapper to obtain a process' output that returns the output generated
+# by running the given command if it exits normally, and streams that
+# output to log.debug and calls die or the given error callback if it
+# does not.
+@imports(_from='mozbuild.configure.util', _import='LineIO')
+@imports(_from='mozbuild.shellutil', _import='quote')
+def check_cmd_output(*args, **kwargs):
+    onerror = kwargs.pop('onerror', None)
+
     with log.queue_debug():
-        log.debug('Executing: `%s`', quote(*args))
-        proc = subprocess.Popen(args, stdout=subprocess.PIPE,
-                                stderr=subprocess.PIPE, **kwargs)
-        stdout, stderr = proc.communicate()
-        retcode = proc.wait()
+        retcode, stdout, stderr = get_cmd_output(*args, **kwargs)
         if retcode == 0:
             return stdout
 
         log.debug('The command returned non-zero exit status %d.',
                   retcode)
         for out, desc in ((stdout, 'output'), (stderr, 'error output')):
             if out:
                 log.debug('Its %s was:', desc)