Bug 1562953 - update cctools-port; r=mshal
☠☠ backed out by eea7be13a029 ☠ ☠
authorNathan Froyd <froydnj@mozilla.com>
Tue, 09 Jul 2019 04:59:37 +0000
changeset 542318 43e086ced66f1d2acf661a9251e5963e58e32cca
parent 542317 624428794690f8065e6a0b3dbffbe524f28120ca
child 542319 8f1f4702a5c23ffabc82130854676e0daca2930a
push id11848
push userffxbld-merge
push dateMon, 26 Aug 2019 19:26:25 +0000
treeherdermozilla-beta@9b31bfdfac10 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmshal
bugs1562953
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1562953 - update cctools-port; r=mshal We need a fix from `cctools-port` master for cross-language LTO builds to work properly on the Mac. Rather than cherry-picking yet another commit, which would have to deal with a updated `ld64` upstream, we've opted to go ahead and update directly to upstream. This choice brings about some significant build changes, as TAPI support has moved to a different library that is not easily buildable directly. Differential Revision: https://phabricator.services.mozilla.com/D36636
build/macosx/cross-mozconfig.common
taskcluster/ci/toolchain/cctools-port.yml
taskcluster/scripts/misc/build-cctools-port.sh
taskcluster/scripts/misc/build-clang-8-linux-macosx-cross.sh
taskcluster/scripts/misc/build-clang-macosx.sh
taskcluster/scripts/misc/build-clang-tidy-macosx.sh
taskcluster/scripts/misc/build-gn-macosx.sh
taskcluster/scripts/misc/wr-macos-cross-build-setup.sh
--- a/build/macosx/cross-mozconfig.common
+++ b/build/macosx/cross-mozconfig.common
@@ -5,17 +5,18 @@
 MOZ_AUTOMATION_L10N_CHECK=0
 
 . "$topsrcdir/build/mozconfig.common"
 
 # cctools for ld, ar, and other related tools ; dsymutil for rust.
 mk_add_options "export PATH=$topsrcdir/cctools/bin:$topsrcdir/binutils/bin:$topsrcdir/llvm-dsymutil/bin:$PATH"
 
 # ld needs libLTO.so from llvm
-mk_add_options "export LD_LIBRARY_PATH=$topsrcdir/clang/lib"
+# ld also needs libtapi.so from cctools itself
+mk_add_options "export LD_LIBRARY_PATH=$topsrcdir/clang/lib:$topsrcdir/cctools/lib"
 
 # This SDK was copied from a local XCode install and uploaded to tooltool.
 # Generate the tarball by running this command with the proper SDK version:
 #   sdk_path=$(xcrun --sdk macosx10.12 --show-sdk-path)
 #   tar -C $(dirname ${sdk_path}) -cHjf /tmp/$(basename ${sdk_path}).tar.bz2 $(basename ${sdk_path})
 # Upload the resulting tarball from /tmp to tooltool, and change the entry in
 # `browser/config/tooltool-manifests/macosx64/cross-releng.manifest`.
 CROSS_SYSROOT=$topsrcdir/MacOSX10.11.sdk
--- a/taskcluster/ci/toolchain/cctools-port.yml
+++ b/taskcluster/ci/toolchain/cctools-port.yml
@@ -14,8 +14,9 @@ job-defaults:
 linux64-cctools-port:
     treeherder:
         symbol: TL(cctools)
     run:
         script: build-cctools-port.sh
         toolchain-artifact: public/build/cctools.tar.xz
     toolchains:
         - linux64-clang-8
+        - linux64-binutils
--- a/taskcluster/scripts/misc/build-cctools-port.sh
+++ b/taskcluster/scripts/misc/build-cctools-port.sh
@@ -8,46 +8,98 @@ set -x -e -v
 
 # This script is for building cctools (Apple's binutils) for Linux using
 # cctools-port (https://github.com/tpoechtrager/cctools-port).
 WORKSPACE=$HOME/workspace
 UPLOAD_DIR=$HOME/artifacts
 
 # Repository info
 : CROSSTOOL_PORT_REPOSITORY    ${CROSSTOOL_PORT_REPOSITORY:=https://github.com/tpoechtrager/cctools-port}
-: CROSSTOOL_PORT_REV           ${CROSSTOOL_PORT_REV:=8e9c3f2506b51cf56725eaa60b6e90e240e249ca}
+: CROSSTOOL_PORT_REV           ${CROSSTOOL_PORT_REV:=3f979bbcd7ee29d79fb93f829edf3d1d16441147}
+: LIBTAPI_REPOSITORY           ${LIBTAPI_REPOSITORY:=https://github.com/tpoechtrager/apple-libtapi}
+: LIBTAPI_REV                  ${LIBTAPI_REV:=3efb201881e7a76a21e0554906cf306432539cef}
 
-# Set some crosstools-port directories
+# Set some crosstools-port and libtapi directories
 CROSSTOOLS_SOURCE_DIR=$WORKSPACE/crosstools-port
 CROSSTOOLS_CCTOOLS_DIR=$CROSSTOOLS_SOURCE_DIR/cctools
 CROSSTOOLS_BUILD_DIR=$WORKSPACE/cctools
+LIBTAPI_SOURCE_DIR=$WORKSPACE/apple-libtapi
+LIBTAPI_BUILD_DIR=$WORKSPACE/libtapi-build
 CLANG_DIR=$WORKSPACE/build/src/clang
 
 # Create our directories
-mkdir -p $CROSSTOOLS_BUILD_DIR
+mkdir -p $CROSSTOOLS_BUILD_DIR $LIBTAPI_BUILD_DIR
 
-git clone --no-checkout $CROSSTOOL_PORT_REPOSITORY $CROSSTOOLS_SOURCE_DIR
+# Check for checkouts first to make interactive usage on taskcluster nicer.
+if [ ! -d $CROSSTOOLS_SOURCE_DIR ]; then
+    git clone --no-checkout $CROSSTOOL_PORT_REPOSITORY $CROSSTOOLS_SOURCE_DIR
+fi
 cd $CROSSTOOLS_SOURCE_DIR
 git checkout $CROSSTOOL_PORT_REV
-# Cherry pick two fixes for LTO.
-git cherry-pick -n 82381f5038a340025ae145745ae5b325cd1b749a
-git cherry-pick -n 328c7371008a854af30823adcd4ec1e763054a1d
-echo "Building from commit hash `git rev-parse $CROSSTOOL_PORT_REV`..."
+echo "Building cctools from commit hash `git rev-parse $CROSSTOOL_PORT_REV`..."
+
+if [ ! -d $LIBTAPI_SOURCE_DIR ]; then
+    git clone --no-checkout $LIBTAPI_REPOSITORY $LIBTAPI_SOURCE_DIR
+fi
+cd $LIBTAPI_SOURCE_DIR
+git checkout $LIBTAPI_REV
+echo "Building libtapi from commit hash `git rev-parse $LIBTAPI_REV`..."
 
 # Fetch clang from tooltool
 cd $WORKSPACE/build/src
 . taskcluster/scripts/misc/tooltool-download.sh
 
-# Configure crosstools-port
-cd $CROSSTOOLS_CCTOOLS_DIR
+export PATH="$WORKSPACE/build/src/binutils/bin:$PATH"
+
+# Common setup for libtapi and cctools
 export CC=$CLANG_DIR/bin/clang
 export CXX=$CLANG_DIR/bin/clang++
+# TODO: bug 1357317 to avoid the LD_LIBRARY_PATH.
+# We also need this LD_LIBRARY_PATH at build time, since tapi builds bits of
+# clang build tools, and then executes those tools.
+export LD_LIBRARY_PATH=$CLANG_DIR/lib
+
+# Build libtapi; the included build.sh is not sufficient for our purposes.
+cd $LIBTAPI_BUILD_DIR
+
+# Values taken from build.sh
+TAPI_REPOSITORY=tapi-1000.10.8
+TAPI_VERSION=10.0.0
+
+INCLUDE_FIX="-I $LIBTAPI_SOURCE_DIR/src/llvm/projects/clang/include -I $PWD/projects/clang/include"
+
+cmake $LIBTAPI_SOURCE_DIR/src/llvm \
+      -GNinja \
+      -DCMAKE_CXX_FLAGS="$INCLUDE_FIX" \
+      -DLLVM_INCLUDE_TESTS=OFF \
+      -DCMAKE_BUILD_TYPE=RELEASE \
+      -DCMAKE_INSTALL_PREFIX=$CROSSTOOLS_BUILD_DIR \
+      -DLLVM_TARGETS_TO_BUILD="X86;ARM;AArch64" \
+      -DTAPI_REPOSITORY_STRING=$TAPI_REPOSITORY \
+      -DTAPI_FULL_VERSION=$TAPI_VERSION
+
+ninja clangBasic
+ninja libtapi install-libtapi install-tapi-headers
+
+# Setup LDFLAGS late so run-at-build-time tools in the basic clang build don't
+# pick up the possibly-incompatible libstdc++ from clang.
 export LDFLAGS="-lpthread -Wl,-rpath-link,$CLANG_DIR/lib"
+
+# Configure crosstools-port
+cd $CROSSTOOLS_CCTOOLS_DIR
+# Force re-libtoolization to overwrite files with the new libtool bits.
+perl -pi -e 's/(LIBTOOLIZE -c)/\1 -f/' autogen.sh
 ./autogen.sh
-./configure --prefix=$CROSSTOOLS_BUILD_DIR --target=x86_64-apple-darwin --with-llvm-config=$CLANG_DIR/bin/llvm-config
+./configure \
+    --prefix=$CROSSTOOLS_BUILD_DIR \
+    --target=x86_64-apple-darwin \
+    --with-llvm-config=$CLANG_DIR/bin/llvm-config \
+    --enable-lto-support \
+    --enable-tapi-support \
+    --with-libtapi=$CROSSTOOLS_BUILD_DIR
 
 # Build cctools
 make -j `nproc --all` install
 strip $CROSSTOOLS_BUILD_DIR/bin/*
 # cctools-port doesn't include dsymutil but clang will need to find it.
 cp $CLANG_DIR/bin/dsymutil $CROSSTOOLS_BUILD_DIR/bin/x86_64-apple-darwin-dsymutil
 # various build scripts based on cmake want to find `lipo` without a prefix
 cp $CROSSTOOLS_BUILD_DIR/bin/x86_64-apple-darwin-lipo $CROSSTOOLS_BUILD_DIR/bin/lipo
--- a/taskcluster/scripts/misc/build-clang-8-linux-macosx-cross.sh
+++ b/taskcluster/scripts/misc/build-clang-8-linux-macosx-cross.sh
@@ -6,18 +6,18 @@ set -x -e -v
 WORKSPACE=$HOME/workspace
 HOME_DIR=$WORKSPACE/build
 UPLOAD_DIR=$HOME/artifacts
 
 cd $HOME_DIR/src
 
 . taskcluster/scripts/misc/tooltool-download.sh
 
-# ld needs libLTO.so from llvm
-export LD_LIBRARY_PATH=$HOME_DIR/src/clang/lib
+# ld needs libLTO.so from llvm and libtapi.so from cctools.
+export LD_LIBRARY_PATH=$HOME_DIR/src/clang/lib:$HOME_DIR/src/cctools/lib
 # these variables are used in build-clang.py
 export CROSS_CCTOOLS_PATH=$HOME_DIR/src/cctools
 export CROSS_SYSROOT=$HOME_DIR/src/MacOSX10.11.sdk
 export PATH=$PATH:$CROSS_CCTOOLS_PATH/bin
 
 # gets a bit too verbose here
 set +x
 
--- a/taskcluster/scripts/misc/build-clang-macosx.sh
+++ b/taskcluster/scripts/misc/build-clang-macosx.sh
@@ -5,18 +5,18 @@ set -x -e -v
 WORKSPACE=$HOME/workspace
 HOME_DIR=$WORKSPACE/build
 UPLOAD_DIR=$HOME/artifacts
 
 cd $HOME_DIR/src
 
 . taskcluster/scripts/misc/tooltool-download.sh
 
-# ld needs libLTO.so from llvm
-export LD_LIBRARY_PATH=$HOME_DIR/src/clang/lib
+# ld needs libLTO.so from llvm and libtapi.so from cctools.
+export LD_LIBRARY_PATH=$HOME_DIR/src/clang/lib:$HOME_DIR/src/cctools/lib
 # these variables are used in build-clang.py
 export CROSS_CCTOOLS_PATH=$HOME_DIR/src/cctools
 export CROSS_SYSROOT=$HOME_DIR/src/MacOSX10.11.sdk
 export PATH=$PATH:$CROSS_CCTOOLS_PATH/bin
 
 # gets a bit too verbose here
 set +x
 
--- a/taskcluster/scripts/misc/build-clang-tidy-macosx.sh
+++ b/taskcluster/scripts/misc/build-clang-tidy-macosx.sh
@@ -5,18 +5,18 @@ set -x -e -v
 WORKSPACE=$HOME/workspace
 HOME_DIR=$WORKSPACE/build
 UPLOAD_DIR=$HOME/artifacts
 
 cd $HOME_DIR/src
 
 . taskcluster/scripts/misc/tooltool-download.sh
 
-# ld needs libLTO.so from llvm
-export LD_LIBRARY_PATH=$HOME_DIR/src/clang/lib
+# ld needs libLTO.so from llvm and libtapi.so from cctools.
+export LD_LIBRARY_PATH=$HOME_DIR/src/clang/lib:$HOME_DIR/src/cctools/lib
 # these variables are used in build-clang.py
 export CROSS_CCTOOLS_PATH=$HOME_DIR/src/cctools
 export CROSS_SYSROOT=$HOME_DIR/src/MacOSX10.11.sdk
 export PATH=$PATH:$CROSS_CCTOOLS_PATH/bin
 
 # gets a bit too verbose here
 set +x
 
--- a/taskcluster/scripts/misc/build-gn-macosx.sh
+++ b/taskcluster/scripts/misc/build-gn-macosx.sh
@@ -5,17 +5,17 @@ set -e -v
 
 WORKSPACE=$HOME/workspace
 UPLOAD_DIR=$HOME/artifacts
 COMPRESS_EXT=xz
 
 CROSS_CCTOOLS_PATH=$WORKSPACE/build/src/cctools
 CROSS_SYSROOT=$WORKSPACE/build/src/MacOSX10.11.sdk
 
-export LD_LIBRARY_PATH=$WORKSPACE/build/src/clang/lib
+export LD_LIBRARY_PATH=$WORKSPACE/build/src/clang/lib:$WORKSPACE/build/src/cctools/lib
 export CC=$WORKSPACE/build/src/clang/bin/clang
 export CXX=$WORKSPACE/build/src/clang/bin/clang++
 export AR=$WORKSPACE/build/src/clang/bin/llvm-ar
 export CFLAGS="-target x86_64-apple-darwin -mlinker-version=137 -B ${CROSS_CCTOOLS_PATH}/bin -isysroot ${CROSS_SYSROOT} -I${CROSS_SYSROOT}/usr/include -iframework ${CROSS_SYSROOT}/System/Library/Frameworks"
 export CXXFLAGS="-stdlib=libc++ ${CFLAGS}"
 export LDFLAGS="${CXXFLAGS} -Wl,-syslibroot,${CROSS_SYSROOT} -Wl,-dead_strip"
 
 # We patch tools/gn/bootstrap/bootstrap.py to detect this.
--- a/taskcluster/scripts/misc/wr-macos-cross-build-setup.sh
+++ b/taskcluster/scripts/misc/wr-macos-cross-build-setup.sh
@@ -46,17 +46,18 @@ popd
 # llvmpipe anyway since we only use the softpipe driver. If for whatever
 # reason we need to add clang/bin to the path here, we should be able to
 # instead set LLVM_CONFIG=no to disable llvmpipe, but that might impact
 # other parts of the build.
 export PATH="${TOOLS_DIR}/rustc/bin:${TOOLS_DIR}/cctools/bin:${TOOLS_DIR}/llvm-dsymutil/bin:${PATH}"
 
 # The x86_64-darwin11-ld linker from cctools requires libraries provided
 # by clang, so we need to set LD_LIBRARY_PATH for that to work.
-export LD_LIBRARY_PATH="${CLANGDIR}/lib:${LD_LIBRARY_PATH}"
+# The linker also needs libtapi.so from cctools.
+export LD_LIBRARY_PATH="${CLANGDIR}/lib:${TOOLS_DIR}/cctools/lib:${LD_LIBRARY_PATH}"
 
 # Tell the configure script where to find zlib, because otherwise it tries
 # to use pkg-config to find it, which fails (no .pc file in the macos SDK).
 export ZLIB_CFLAGS="-I${MACOS_SYSROOT}/usr/include"
 export ZLIB_LIBS="-L${MACOS_SYSROOT}/usr/lib -lz"
 
 # Set up compiler and flags for cross-compile
 LDPATH="${TOOLS_DIR}/cctools/bin/${TARGET_TRIPLE}-ld"