Bug 1492663 - Upgrade most CI builds to clang 7 r=froydnj
☠☠ backed out by 19b215c7a61e ☠ ☠
authorMike Hommey <mh+mozilla@glandium.org>
Thu, 27 Sep 2018 15:33:42 +0000
changeset 495507 d36c0c30cc03af21ff3b5f24c1a8bb8bcaa6926e
parent 495506 e91146cb4ee45cb2c0dfdfa5cce64394404467c3
child 495508 2a6ec2aae9fdafeb0efd81310a9f16ba5ccfd6dc
push id9984
push userffxbld-merge
push dateMon, 15 Oct 2018 21:07:35 +0000
treeherdermozilla-beta@183d27ea8570 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfroydnj
bugs1492663, 1471905
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1492663 - Upgrade most CI builds to clang 7 r=froydnj The cctools-port linker links against libraries from clang (for LTO), which have different SONAMEs depending on the clang version. Which means the linker needs to be used along the same version of clang it was built against. Thus we also make it depend on linux64-clang-7. But changing the dependency is not enough, cf. bug 1471905, so also touch its build script, which it turns out, we need to do anyways because llvm-dsymutil was renamed to dsymutil. Relatedly, all toolchains that are built using cctools-port need to use linux64-clang-7 too. Building compiler-rt 7 with the OSX 10.11 SDK fails because of some newer APIs being used in compiler-rt for xray, but this is not a feature we use, so disable that. Differential Revision: https://phabricator.services.mozilla.com/D6766
build/build-clang/build-clang.py
build/build-clang/clang-6-macosx64.json
build/build-clang/clang-7-macosx64.json
build/build-clang/compiler-rt-no-codesign.patch
taskcluster/ci/toolchain/linux.yml
taskcluster/ci/toolchain/macosx.yml
taskcluster/scripts/misc/build-cctools-port.sh
taskcluster/scripts/misc/build-clang-6-linux-macosx-cross.sh
taskcluster/scripts/misc/build-clang-7-linux-macosx-cross.sh
taskcluster/scripts/misc/build-clang-macosx.sh
--- a/build/build-clang/build-clang.py
+++ b/build/build-clang/build-clang.py
@@ -230,16 +230,18 @@ def build_one_stage(cc, cxx, asm, ld, ar
             cmake_args += ["-DCMAKE_RANLIB=%s" % slashify_path(ranlib)]
         if libtool is not None:
             cmake_args += ["-DCMAKE_LIBTOOL=%s" % slashify_path(libtool)]
         if osx_cross_compile:
             cmake_args += [
                 "-DCMAKE_SYSTEM_NAME=Darwin",
                 "-DCMAKE_SYSTEM_VERSION=10.10",
                 "-DLLVM_ENABLE_THREADS=OFF",
+                # Xray requires a OSX 10.12 SDK (https://bugs.llvm.org/show_bug.cgi?id=38959)
+                "-DCOMPILER_RT_BUILD_XRAY=OFF",
                 "-DLIBCXXABI_LIBCXX_INCLUDES=%s" % libcxx_include_dir,
                 "-DCMAKE_OSX_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
                 "-DCMAKE_FIND_ROOT_PATH=%s" % slashify_path(os.getenv("CROSS_CCTOOLS_PATH")), # noqa
                 "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER",
                 "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY",
                 "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY",
                 "-DCMAKE_MACOSX_RPATH=ON",
                 "-DCMAKE_OSX_ARCHITECTURES=x86_64",
deleted file mode 100644
--- a/build/build-clang/clang-6-macosx64.json
+++ /dev/null
@@ -1,31 +0,0 @@
-{
-    "llvm_revision": "335538",
-    "stages": "1",
-    "build_libcxx": true,
-    "build_type": "Release",
-    "assertions": false,
-    "osx_cross_compile": true,
-    "llvm_repo": "https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_601/final",
-    "clang_repo": "https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_601/final",
-    "lld_repo": "https://llvm.org/svn/llvm-project/lld/tags/RELEASE_601/final",
-    "compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/tags/RELEASE_601/final",
-    "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/tags/RELEASE_601/final",
-    "libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/tags/RELEASE_601/final",
-    "python_path": "/usr/bin/python2.7",
-    "gcc_dir": "/builds/worker/workspace/build/src/gcc",
-    "cc": "/builds/worker/workspace/build/src/clang/bin/clang",
-    "cxx": "/builds/worker/workspace/build/src/clang/bin/clang++",
-    "as": "/builds/worker/workspace/build/src/clang/bin/clang",
-    "ar": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin11-ar",
-    "ranlib": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin11-ranlib",
-    "libtool": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin11-libtool",
-    "ld": "/builds/worker/workspace/build/src/clang/bin/clang",
-    "patches": [
-      "static-llvm-symbolizer.patch",
-      "compiler-rt-cross-compile.patch",
-      "compiler-rt-no-codesign.patch",
-      "r322401.patch",
-      "r325356.patch",
-      "r339636.patch"
-    ]
-}
new file mode 100644
--- /dev/null
+++ b/build/build-clang/clang-7-macosx64.json
@@ -0,0 +1,28 @@
+{
+    "llvm_revision": "342383",
+    "stages": "1",
+    "build_libcxx": true,
+    "build_type": "Release",
+    "assertions": false,
+    "osx_cross_compile": true,
+    "llvm_repo": "https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_700/final",
+    "clang_repo": "https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_700/final",
+    "lld_repo": "https://llvm.org/svn/llvm-project/lld/tags/RELEASE_700/final",
+    "compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/tags/RELEASE_700/final",
+    "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/tags/RELEASE_700/final",
+    "libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/tags/RELEASE_700/final",
+    "python_path": "/usr/bin/python2.7",
+    "gcc_dir": "/builds/worker/workspace/build/src/gcc",
+    "cc": "/builds/worker/workspace/build/src/clang/bin/clang",
+    "cxx": "/builds/worker/workspace/build/src/clang/bin/clang++",
+    "as": "/builds/worker/workspace/build/src/clang/bin/clang",
+    "ar": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin11-ar",
+    "ranlib": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin11-ranlib",
+    "libtool": "/builds/worker/workspace/build/src/cctools/bin/x86_64-apple-darwin11-libtool",
+    "ld": "/builds/worker/workspace/build/src/clang/bin/clang",
+    "patches": [
+      "static-llvm-symbolizer.patch",
+      "compiler-rt-cross-compile.patch",
+      "compiler-rt-no-codesign.patch"
+    ]
+}
--- a/build/build-clang/compiler-rt-no-codesign.patch
+++ b/build/build-clang/compiler-rt-no-codesign.patch
@@ -1,21 +1,21 @@
 Disable codesign for macosx cross-compile toolchain. Codesign only works on OSX.
 
 Index: cmake/Modules/AddCompilerRT.cmake
 ===================================================================
---- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake	(revision 312553)
+--- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake	(revision 342374)
 +++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake	(working copy)
-@@ -224,14 +224,6 @@
+@@ -290,14 +290,6 @@
          set_target_properties(${libname} PROPERTIES IMPORT_PREFIX "")
          set_target_properties(${libname} PROPERTIES IMPORT_SUFFIX ".lib")
        endif()
 -      if(APPLE)
 -        # Ad-hoc sign the dylibs
 -        add_custom_command(TARGET ${libname}
 -          POST_BUILD  
 -          COMMAND codesign --sign - $<TARGET_FILE:${libname}>
 -          WORKING_DIRECTORY ${COMPILER_RT_LIBRARY_OUTPUT_DIR}
 -        )
 -      endif()
      endif()
      install(TARGETS ${libname}
-       ARCHIVE DESTINATION ${COMPILER_RT_LIBRARY_INSTALL_DIR}
+       ARCHIVE DESTINATION ${install_dir_${libname}}
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@@ -39,17 +39,16 @@ linux64-clang-6:
         max-run-time: 7200
     run:
         using: toolchain-script
         script: build-clang-6-linux.sh
         resources:
             - 'build/build-clang/build-clang.py'
             - 'build/build-clang/clang-6-linux64.json'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
-        toolchain-alias: linux64-clang
         toolchain-artifact: public/build/clang.tar.xz
     toolchains:
         - linux64-gcc-4.9
 
 linux64-clang-7:
     description: "Clang 7 toolchain build"
     treeherder:
         kind: build
@@ -61,16 +60,17 @@ linux64-clang-7:
         max-run-time: 7200
     run:
         using: toolchain-script
         script: build-clang-7-linux.sh
         resources:
             - 'build/build-clang/build-clang.py'
             - 'build/build-clang/clang-7-linux64.json'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-alias: linux64-clang
         toolchain-artifact: public/build/clang.tar.xz
     toolchains:
         - linux64-gcc-4.9
 
 linux64-clang-7-mingw-x86:
     description: "MinGW-Clang 7 x86 toolchain build"
     treeherder:
         kind: build
@@ -113,41 +113,41 @@ linux64-clang-7-mingw-x64:
         resources:
             - 'build/build-clang/build-clang.py'
             - 'build/build-clang/clang-7-mingw.json'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/clangmingw.tar.xz
     toolchains:
         - linux64-gcc-4.9
 
-linux64-clang-6-macosx-cross:
-    description: "Clang 6 toolchain build with MacOS Compiler RT libs"
+linux64-clang-7-macosx-cross:
+    description: "Clang 7 toolchain build with MacOS Compiler RT libs"
     treeherder:
         kind: build
         platform: toolchains/opt
-        symbol: TL(clang6-macosx-cross)
+        symbol: TL(clang7-macosx-cross)
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 3600
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
     run:
         using: toolchain-script
-        script: build-clang-6-linux-macosx-cross.sh
+        script: build-clang-7-linux-macosx-cross.sh
         resources:
             - 'build/build-clang/build-clang.py'
-            - 'build/build-clang/clang-6-macosx64.json'
+            - 'build/build-clang/clang-7-macosx64.json'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-alias: linux64-clang-macosx-cross
         toolchain-artifact: public/build/clang.tar.xz
         tooltool-downloads: internal
     toolchains:
         - linux64-cctools-port
-        - linux64-clang-6
+        - linux64-clang-7
         - linux64-gcc-4.9
 
 linux64-clang-tidy:
     description: "Clang-tidy build"
     index:
         product: static-analysis
         job-name: linux64-clang-tidy
     treeherder:
@@ -349,17 +349,17 @@ linux64-cctools-port:
         max-run-time: 1800
     run:
         using: toolchain-script
         script: build-cctools-port.sh
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/cctools.tar.xz
     toolchains:
-        - linux64-clang-6
+        - linux64-clang-7
 
 linux64-hfsplus:
     description: "hfsplus toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(hfs+)
         tier: 1
--- a/taskcluster/ci/toolchain/macosx.yml
+++ b/taskcluster/ci/toolchain/macosx.yml
@@ -22,22 +22,22 @@ macosx64-clang:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
     run:
         using: toolchain-script
         script: build-clang-macosx.sh
         tooltool-downloads: internal
         resources:
             - 'build/build-clang/build-clang.py'
-            - 'build/build-clang/clang-6-macosx64.json'
+            - 'build/build-clang/clang-7-macosx64.json'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/clang.tar.xz
     toolchains:
         - linux64-cctools-port
-        - linux64-clang-6
+        - linux64-clang-7
         - linux64-gcc-4.9
         - linux64-node
 
 macosx64-clang-tidy:
     description: "Clang-tidy build"
     index:
         product: static-analysis
         job-name: macosx64-clang-tidy
@@ -58,17 +58,17 @@ macosx64-clang-tidy:
         resources:
             - 'build/clang-plugin/**'
             - 'build/build-clang/build-clang.py'
             - 'build/build-clang/clang-tidy-macosx64.json'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/clang-tidy.tar.xz
     toolchains:
         - linux64-cctools-port
-        - linux64-clang-6
+        - linux64-clang-7
         - linux64-gcc-4.9
         - linux64-node
 
 macosx64-cctools-port:
     description: "cctools-port toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
@@ -83,17 +83,17 @@ macosx64-cctools-port:
         using: toolchain-script
         script: build-cctools-port-macosx.sh
         tooltool-downloads: internal
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/cctools.tar.bz2
     toolchains:
         - linux64-cctools-port
-        - linux64-clang-6
+        - linux64-clang-7
         - linux64-node
 
 macosx64-gn:
     description: "gn toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TM(gn)
@@ -108,17 +108,17 @@ macosx64-gn:
         script: build-gn-macosx.sh
         tooltool-downloads: internal
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
             - 'taskcluster/scripts/misc/build-gn-common.sh'
         toolchain-artifact: public/build/gn.tar.xz
     toolchains:
         - linux64-cctools-port
-        - linux64-clang-6
+        - linux64-clang-7
         - linux64-node
 
 macosx64-node:
     description: "Node repack toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TM(node)
--- a/taskcluster/scripts/misc/build-cctools-port.sh
+++ b/taskcluster/scripts/misc/build-cctools-port.sh
@@ -43,13 +43,13 @@ export CXX=$CLANG_DIR/bin/clang++
 export LDFLAGS="-lpthread -Wl,-rpath-link,$CLANG_DIR/lib"
 ./autogen.sh
 ./configure --prefix=$CROSSTOOLS_BUILD_DIR --target=x86_64-apple-darwin11 --with-llvm-config=$CLANG_DIR/bin/llvm-config
 
 # Build cctools
 make -j `nproc --all` install
 strip $CROSSTOOLS_BUILD_DIR/bin/*
 # cctools-port doesn't include dsymutil but clang will need to find it.
-cp $CLANG_DIR/bin/llvm-dsymutil $CROSSTOOLS_BUILD_DIR/bin/x86_64-apple-darwin11-dsymutil
+cp $CLANG_DIR/bin/dsymutil $CROSSTOOLS_BUILD_DIR/bin/x86_64-apple-darwin11-dsymutil
 
 # Put a tarball in the artifacts dir
 mkdir -p $UPLOAD_DIR
 tar cJf $UPLOAD_DIR/cctools.tar.xz -C $CROSSTOOLS_BUILD_DIR/.. `basename $CROSSTOOLS_BUILD_DIR`
deleted file mode 100755
--- a/taskcluster/scripts/misc/build-clang-6-linux-macosx-cross.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/bin/bash
-set -x -e -v
-
-# This script is for building clang for Mac OS X targets on a Linux host,
-# including native Mac OS X Compiler-RT libraries and llvm-symbolizer.
-WORKSPACE=$HOME/workspace
-HOME_DIR=$WORKSPACE/build
-UPLOAD_DIR=$HOME/artifacts
-
-cd $HOME_DIR/src
-
-. taskcluster/scripts/misc/tooltool-download.sh
-
-# ld needs libLTO.so from llvm
-export LD_LIBRARY_PATH=$HOME_DIR/src/clang/lib
-# these variables are used in build-clang.py
-export CROSS_CCTOOLS_PATH=$HOME_DIR/src/cctools
-export CROSS_SYSROOT=$HOME_DIR/src/MacOSX10.11.sdk
-# cmake doesn't allow us to specify a path to lipo on the command line.
-export PATH=$PATH:$CROSS_CCTOOLS_PATH/bin
-ln -sf $CROSS_CCTOOLS_PATH/bin/x86_64-apple-darwin11-lipo $CROSS_CCTOOLS_PATH/bin/lipo
-
-# gets a bit too verbose here
-set +x
-
-cd build/build-clang
-# |mach python| sets up a virtualenv for us!
-../../mach python ./build-clang.py -c clang-6-macosx64.json --skip-tar
-
-# We now have a native macosx64 toolchain.
-# What we want is a native linux64 toolchain which can target macosx64 and use the sanitizer dylibs.
-# Overlay the linux64 toolchain that we used for this build (except llvm-symbolizer).
-(
-cd "$WORKSPACE/moz-toolchain/build/stage1"
-# Need the macosx64 native llvm-symbolizer since this gets shipped with sanitizer builds
-mv clang/bin/llvm-symbolizer $HOME_DIR/src/clang/bin/
-cp --remove-destination -lr $HOME_DIR/src/clang/* clang/
-tar -c -J -f $HOME_DIR/src/build/build-clang/clang.tar.xz clang
-)
-
-set -x
-
-# Put a tarball in the artifacts dir
-mkdir -p $UPLOAD_DIR
-cp clang.tar.* $UPLOAD_DIR
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/misc/build-clang-7-linux-macosx-cross.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+set -x -e -v
+
+# This script is for building clang for Mac OS X targets on a Linux host,
+# including native Mac OS X Compiler-RT libraries and llvm-symbolizer.
+WORKSPACE=$HOME/workspace
+HOME_DIR=$WORKSPACE/build
+UPLOAD_DIR=$HOME/artifacts
+
+cd $HOME_DIR/src
+
+. taskcluster/scripts/misc/tooltool-download.sh
+
+# ld needs libLTO.so from llvm
+export LD_LIBRARY_PATH=$HOME_DIR/src/clang/lib
+# these variables are used in build-clang.py
+export CROSS_CCTOOLS_PATH=$HOME_DIR/src/cctools
+export CROSS_SYSROOT=$HOME_DIR/src/MacOSX10.11.sdk
+# cmake doesn't allow us to specify a path to lipo on the command line.
+export PATH=$PATH:$CROSS_CCTOOLS_PATH/bin
+ln -sf $CROSS_CCTOOLS_PATH/bin/x86_64-apple-darwin11-lipo $CROSS_CCTOOLS_PATH/bin/lipo
+
+# gets a bit too verbose here
+set +x
+
+cd build/build-clang
+# |mach python| sets up a virtualenv for us!
+../../mach python ./build-clang.py -c clang-7-macosx64.json --skip-tar
+
+# We now have a native macosx64 toolchain.
+# What we want is a native linux64 toolchain which can target macosx64 and use the sanitizer dylibs.
+# Overlay the linux64 toolchain that we used for this build (except llvm-symbolizer).
+(
+cd "$WORKSPACE/moz-toolchain/build/stage1"
+# Need the macosx64 native llvm-symbolizer since this gets shipped with sanitizer builds
+mv clang/bin/llvm-symbolizer $HOME_DIR/src/clang/bin/
+cp --remove-destination -lr $HOME_DIR/src/clang/* clang/
+tar -c -J -f $HOME_DIR/src/build/build-clang/clang.tar.xz clang
+)
+
+set -x
+
+# Put a tarball in the artifacts dir
+mkdir -p $UPLOAD_DIR
+cp clang.tar.* $UPLOAD_DIR
--- a/taskcluster/scripts/misc/build-clang-macosx.sh
+++ b/taskcluster/scripts/misc/build-clang-macosx.sh
@@ -19,15 +19,15 @@ export CROSS_SYSROOT=$HOME_DIR/src/MacOS
 export PATH=$PATH:$CROSS_CCTOOLS_PATH/bin
 ln -sf $CROSS_CCTOOLS_PATH/bin/x86_64-apple-darwin11-lipo $CROSS_CCTOOLS_PATH/bin/lipo
 
 # gets a bit too verbose here
 set +x
 
 cd build/build-clang
 # |mach python| sets up a virtualenv for us!
-../../mach python ./build-clang.py -c clang-6-macosx64.json
+../../mach python ./build-clang.py -c clang-7-macosx64.json
 
 set -x
 
 # Put a tarball in the artifacts dir
 mkdir -p $UPLOAD_DIR
 cp clang.tar.* $UPLOAD_DIR