Bug 1492037 - Build clang with LLVM as a shared library. r=froydnj
authorMike Hommey <mh+mozilla@glandium.org>
Mon, 10 Sep 2018 07:41:35 +0900
changeset 493327 7e05c2752c5a2a16082854a01e76e53c1b03659b
parent 493326 0e28dd35739698fc6794ca4aa218975c512acef1
child 493328 ebeba937ca2a01df86d98089f2368bb975182dcc
child 493340 3d337c5ca45769a63664ddac4e8e1ecf5af9b972
push id9984
push userffxbld-merge
push dateMon, 15 Oct 2018 21:07:35 +0000
treeherdermozilla-beta@183d27ea8570 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfroydnj
bugs1492037
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1492037 - Build clang with LLVM as a shared library. r=froydnj Doing so changes the size of the compressed toolchain archive from ~280M to ~120M, and the decompressed size from ~1500M to ~675M. This will reduce the overhead of decompression during builds. As we ship llvm-symbolizer as part of ASan builds, we do need it to still statically link against LLVM, which we do with a small patch. With LLVM as a shared library, libLTO, which is used by cctools-port for the linker, is dynamically linked to LLVM, and the cctools-port configure script fails to link against libLTO. So we add a -rpath-link to make it find the LLVM library. This happens to force a rebuild of cctools-port, but for future cases where we might need a rebuild because of some clang changes, we add a comment to ease the process, and avoid a newer cctools-port taking the cache spot of an older one. Ideally, mac cctools-port would need something similar, but it needs a mac libLTO.dylib, which is not there anyways (and the mac cctools-port thus already didn't support LTO). Also, with LLVM built as a shared library, all its symbols are exported with a LLVM_x.y version. Combined with -static-libstdc++ that is used during the clang build, this causes problems (see https://bugzilla.mozilla.org/show_bug.cgi?id=1492037#c7). But it turns out things have evolved since -static-libstdc++ has been added to the clang build script, and things work without now, so remove it (as well as -static-libgcc). Differential Revision: https://phabricator.services.mozilla.com/D6117
build/build-clang/build-clang.py
build/build-clang/clang-6-linux64.json
build/build-clang/clang-6-macosx64.json
build/build-clang/clang-7-linux64.json
build/build-clang/static-llvm-symbolizer.patch
taskcluster/scripts/misc/build-cctools-port-macosx.sh
taskcluster/scripts/misc/build-cctools-port.sh
--- a/build/build-clang/build-clang.py
+++ b/build/build-clang/build-clang.py
@@ -5,16 +5,17 @@
 
 import os
 import os.path
 import shutil
 import subprocess
 import platform
 import json
 import argparse
+import fnmatch
 import glob
 import errno
 import re
 from contextlib import contextmanager
 import sys
 import which
 from distutils.dir_util import copy_tree
 
@@ -217,16 +218,19 @@ def build_one_stage(cc, cxx, asm, ld, ar
             "-DLLVM_TOOL_LIBCXX_BUILD=%s" % ("ON" if build_libcxx else "OFF"),
             "-DLIBCXX_LIBCPPABI_VERSION=\"\"",
         ]
         if is_linux():
             cmake_args += ["-DLLVM_BINUTILS_INCDIR=%s/include" % gcc_dir]
         if is_windows():
             cmake_args.insert(-1, "-DLLVM_EXPORT_SYMBOLS_FOR_PLUGINS=ON")
             cmake_args.insert(-1, "-DLLVM_USE_CRT_RELEASE=MT")
+        else:
+            # libllvm as a shared library is not supported on Windows
+            cmake_args += ["-DLLVM_LINK_LLVM_DYLIB=ON"]
         if ranlib is not None:
             cmake_args += ["-DCMAKE_RANLIB=%s" % slashify_path(ranlib)]
         if libtool is not None:
             cmake_args += ["-DCMAKE_LIBTOOL=%s" % slashify_path(libtool)]
         if osx_cross_compile:
             cmake_args += [
                 "-DCMAKE_SYSTEM_NAME=Darwin",
                 "-DCMAKE_SYSTEM_VERSION=10.10",
@@ -361,21 +365,27 @@ def prune_final_dir_for_clang_tidy(final
     re_clang_tidy = re.compile(
         r"^clang-(apply-replacements|format|tidy)(\.exe)?$", re.I)
     for f in glob.glob("%s/bin/*" % final_dir):
         if re_clang_tidy.search(os.path.basename(f)) is None:
             delete(f)
 
     # Keep include/ intact.
 
-    # In lib/, only keep lib/clang/N.M.O/include.
+    # In lib/, only keep lib/clang/N.M.O/include and the LLVM shared library.
     re_ver_num = re.compile(r"^\d+\.\d+\.\d+$", re.I)
     for f in glob.glob("%s/lib/*" % final_dir):
-        if os.path.basename(f) != "clang":
-            delete(f)
+        name = os.path.basename(f)
+        if name == "clang":
+            continue
+        if is_darwin() and name == 'libLLVM.dylib':
+            continue
+        if is_linux() and fnmatch.fnmatch(name, 'libLLVM*.so'):
+            continue
+        delete(f)
     for f in glob.glob("%s/lib/clang/*" % final_dir):
         if re_ver_num.search(os.path.basename(f)) is None:
             delete(f)
     for f in glob.glob("%s/lib/clang/*/*" % final_dir):
         if os.path.basename(f) != "include":
             delete(f)
 
     # Completely remove libexec/, msbuilld-bin and tools, if it exists.
@@ -595,21 +605,21 @@ if __name__ == "__main__":
     if is_darwin():
         extra_cflags = []
         extra_cxxflags = ["-stdlib=libc++"]
         extra_cflags2 = []
         extra_cxxflags2 = ["-stdlib=libc++"]
         extra_asmflags = []
         extra_ldflags = []
     elif is_linux():
-        extra_cflags = ["-static-libgcc"]
-        extra_cxxflags = ["-static-libgcc", "-static-libstdc++"]
+        extra_cflags = []
+        extra_cxxflags = []
         extra_cflags2 = ["-fPIC"]
         # Silence clang's warnings about arguments not being used in compilation.
-        extra_cxxflags2 = ["-fPIC", '-Qunused-arguments', "-static-libstdc++"]
+        extra_cxxflags2 = ["-fPIC", '-Qunused-arguments']
         extra_asmflags = []
         extra_ldflags = []
 
         if 'LD_LIBRARY_PATH' in os.environ:
             os.environ['LD_LIBRARY_PATH'] = ('%s/lib64/:%s' %
                                              (gcc_dir, os.environ['LD_LIBRARY_PATH']))
         else:
             os.environ['LD_LIBRARY_PATH'] = '%s/lib64/' % gcc_dir
--- a/build/build-clang/clang-6-linux64.json
+++ b/build/build-clang/clang-6-linux64.json
@@ -11,15 +11,16 @@
     "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/tags/RELEASE_601/final",
     "libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/tags/RELEASE_601/final",
     "python_path": "/usr/bin/python2.7",
     "gcc_dir": "/builds/worker/workspace/build/src/gcc",
     "cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",
     "cxx": "/builds/worker/workspace/build/src/gcc/bin/g++",
     "as": "/builds/worker/workspace/build/src/gcc/bin/gcc",
     "patches": [
+      "static-llvm-symbolizer.patch",
       "find_symbolizer_linux.patch",
       "r322325.patch",
       "r322401.patch",
       "r325356.patch",
       "r339636.patch"
     ]
 }
--- a/build/build-clang/clang-6-macosx64.json
+++ b/build/build-clang/clang-6-macosx64.json
@@ -16,15 +16,16 @@
     "cc": "/builds/worker/workspace/build/src/clang/bin/clang",
     "cxx": "/builds/worker/workspace/build/src/clang/bin/clang++",
     "as": "/builds/worker/workspace/build/src/clang/bin/clang",
     "ar": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin11-ar",
     "ranlib": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin11-ranlib",
     "libtool": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin11-libtool",
     "ld": "/builds/worker/workspace/build/src/clang/bin/clang",
     "patches": [
+      "static-llvm-symbolizer.patch",
       "compiler-rt-cross-compile.patch",
       "compiler-rt-no-codesign.patch",
       "r322401.patch",
       "r325356.patch",
       "r339636.patch"
     ]
 }
--- a/build/build-clang/clang-7-linux64.json
+++ b/build/build-clang/clang-7-linux64.json
@@ -11,12 +11,13 @@
     "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/tags/RELEASE_700/final",
     "libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/tags/RELEASE_700/final",
     "python_path": "/usr/bin/python2.7",
     "gcc_dir": "/builds/worker/workspace/build/src/gcc",
     "cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",
     "cxx": "/builds/worker/workspace/build/src/gcc/bin/g++",
     "as": "/builds/worker/workspace/build/src/gcc/bin/gcc",
     "patches": [
+      "static-llvm-symbolizer.patch",
       "find_symbolizer_linux.patch",
       "rename_gcov_flush.patch"
     ]
 }
new file mode 100644
--- /dev/null
+++ b/build/build-clang/static-llvm-symbolizer.patch
@@ -0,0 +1,12 @@
+diff --git a/llvm/tools/llvm-symbolizer/CMakeLists.txt b/llvm/tools/llvm-symbolizer/CMakeLists.txt
+index 8185c296c50..13c7419fa47 100644
+--- a/llvm/tools/llvm-symbolizer/CMakeLists.txt
++++ b/llvm/tools/llvm-symbolizer/CMakeLists.txt
+@@ -13,6 +13,7 @@ set(LLVM_LINK_COMPONENTS
+   )
+ 
+ add_llvm_tool(llvm-symbolizer
++  DISABLE_LLVM_LINK_LLVM_DYLIB
+   llvm-symbolizer.cpp
+   )
+ 
--- a/taskcluster/scripts/misc/build-cctools-port-macosx.sh
+++ b/taskcluster/scripts/misc/build-cctools-port-macosx.sh
@@ -1,9 +1,14 @@
 #!/bin/bash
+
+# cctools sometimes needs to be rebuilt when clang is modified.
+# Until bug 1471905 is addressed, increase the following number
+# when a forced rebuild of cctools is necessary: 1
+
 set -x -e -v
 
 # This script is for building cctools (Apple's binutils) for Mac OS X on
 # Linux using ctools-port (https://github.com/tpoechtrager/cctools-port).
 WORKSPACE=$HOME/workspace
 UPLOAD_DIR=$HOME/artifacts
 
 # Repository info
--- a/taskcluster/scripts/misc/build-cctools-port.sh
+++ b/taskcluster/scripts/misc/build-cctools-port.sh
@@ -1,9 +1,14 @@
 #!/bin/bash
+
+# cctools sometimes needs to be rebuilt when clang is modified.
+# Until bug 1471905 is addressed, increase the following number
+# when a forced rebuild of cctools is necessary: 1
+
 set -x -e -v
 
 # This script is for building cctools (Apple's binutils) for Linux using
 # cctools-port (https://github.com/tpoechtrager/cctools-port).
 WORKSPACE=$HOME/workspace
 UPLOAD_DIR=$HOME/artifacts
 
 # Repository info
@@ -30,17 +35,17 @@ echo "Building from commit hash `git rev
 # Fetch clang from tooltool
 cd $WORKSPACE/build/src
 . taskcluster/scripts/misc/tooltool-download.sh
 
 # Configure crosstools-port
 cd $CROSSTOOLS_CCTOOLS_DIR
 export CC=$CLANG_DIR/bin/clang
 export CXX=$CLANG_DIR/bin/clang++
-export LDFLAGS=-lpthread
+export LDFLAGS="-lpthread -Wl,-rpath-link,$CLANG_DIR/lib"
 ./autogen.sh
 ./configure --prefix=$CROSSTOOLS_BUILD_DIR --target=x86_64-apple-darwin11 --with-llvm-config=$CLANG_DIR/bin/llvm-config
 
 # Build cctools
 make -j `nproc --all` install
 strip $CROSSTOOLS_BUILD_DIR/bin/*
 # cctools-port doesn't include dsymutil but clang will need to find it.
 cp $CLANG_DIR/bin/llvm-dsymutil $CROSSTOOLS_BUILD_DIR/bin/x86_64-apple-darwin11-dsymutil