Bug 1504147 - Build compiler-rt libs for android on arm. r=froydnj
authorChris Manchester <cmanchester@mozilla.com>
Thu, 13 Dec 2018 22:02:01 +0000
changeset 507570 0b16da79b3ed3bf0fec7d0247888ebfebba2a223
parent 507569 aec6d3745e4fc10443ce2e901e29ce423a2e0068
child 507571 f66935a5c80de7fde4812634ba9d4a2fcab97388
push id10547
push userffxbld-merge
push dateMon, 21 Jan 2019 13:03:58 +0000
treeherdermozilla-beta@24ec1916bffe [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfroydnj
bugs1504147
milestone66.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1504147 - Build compiler-rt libs for android on arm. r=froydnj This patch is based on the cmake cache files for Android checked in to the clang repo. Differential Revision: https://phabricator.services.mozilla.com/D14004
build/build-clang/build-clang.py
build/build-clang/clang-7-android.json
taskcluster/ci/build/android.yml
taskcluster/ci/toolchain/linux.yml
taskcluster/scripts/misc/build-clang-7-android.sh
--- a/build/build-clang/build-clang.py
+++ b/build/build-clang/build-clang.py
@@ -181,17 +181,17 @@ def is_linux():
 def is_windows():
     return platform.system() == "Windows"
 
 
 def build_one_stage(cc, cxx, asm, ld, ar, ranlib, libtool,
                     src_dir, stage_dir, package_name, build_libcxx,
                     osx_cross_compile, build_type, assertions,
                     python_path, gcc_dir, libcxx_include_dir,
-                    is_final_stage=False):
+                    is_final_stage=False, android_targets=None):
     if not os.path.exists(stage_dir):
         os.mkdir(stage_dir)
 
     build_dir = stage_dir + "/build"
     inst_dir = stage_dir + "/" + package_name
 
     # cmake doesn't deal well with backslashes in paths.
     def slashify_path(path):
@@ -246,17 +246,65 @@ def build_one_stage(cc, cxx, asm, ld, ar
                 "-DCMAKE_MACOSX_RPATH=ON",
                 "-DCMAKE_OSX_ARCHITECTURES=x86_64",
                 "-DDARWIN_osx_ARCHS=x86_64",
                 "-DDARWIN_osx_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
                 "-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-apple-darwin11"
             ]
         return cmake_args
 
-    cmake_args = cmake_base_args(
+    cmake_args = []
+
+    if is_final_stage and android_targets:
+        cmake_args += [
+            "-DCOMPILER_RT_DEFAULT_TARGET_TRIPLE=%s" % android_targets.keys()[0],
+            "-DLLVM_BUILTIN_TARGETS=%s" % ";".join(android_targets),
+            "-DLLVM_DEFAULT_TARGET_TRIPLE=%s" % android_targets.keys()[0],
+            "-DLLVM_ENABLE_LIBCXX=ON",
+            "-DLLVM_LIBDIR_SUFFIX=64",
+            "-DLLVM_RUNTIME_TARGETS=%s" % ";".join(android_targets),
+        ]
+
+        android_link_flags = "-fuse-ld=lld"
+
+        for target, cfg in android_targets.iteritems():
+            sysroot_dir = cfg["ndk_sysroot"]
+            android_gcc_dir = cfg["ndk_toolchain"]
+            android_include_dirs = cfg["ndk_includes"]
+
+            android_flags = ["-isystem %s" % d for d in android_include_dirs]
+            android_flags += ["--gcc-toolchain=%s" % android_gcc_dir]
+            android_flags += ["-D__ANDROID_API__=16"]
+            rt_c_flags = " ".join(android_flags + cc[1:])
+            rt_cxx_flags = " ".join(android_flags + cxx[1:])
+            rt_asm_flags = " ".join(android_flags + asm[1:])
+
+            cmake_args += [
+                "-DBUILTINS_%s_ANDROID=1" % target,
+                "-DBUILTINS_%s_CMAKE_ASM_FLAGS=%s" % (target, rt_asm_flags),
+                "-DBUILTINS_%s_CMAKE_CXX_FLAGS=%s" % (target, rt_cxx_flags),
+                "-DBUILTINS_%s_CMAKE_C_FLAGS=%s" % (target, rt_c_flags),
+                "-DBUILTINS_%s_CMAKE_EXE_LINKER_FLAGS=%s" % (target, android_link_flags),
+                "-DBUILTINS_%s_CMAKE_SHARED_LINKER_FLAGS=%s" % (target, android_link_flags),
+                "-DBUILTINS_%s_CMAKE_SYSROOT=%s" % (target, sysroot_dir),
+                "-DRUNTIMES_%s_CMAKE_ASM_FLAGS=%s" % (target, rt_asm_flags),
+                "-DRUNTIMES_%s_CMAKE_CXX_FLAGS=%s" % (target, rt_cxx_flags),
+                "-DRUNTIMES_%s_CMAKE_C_FLAGS=%s" % (target, rt_c_flags),
+                "-DRUNTIMES_%s_CMAKE_EXE_LINKER_FLAGS=%s" % (target, android_link_flags),
+                "-DRUNTIMES_%s_CMAKE_SHARED_LINKER_FLAGS=%s" % (target, android_link_flags),
+                "-DRUNTIMES_%s_CMAKE_SYSROOT=%s" % (target, sysroot_dir),
+                "-DRUNTIMES_%s_COMPILER_RT_BUILD_PROFILE=ON" % target,
+                "-DRUNTIMES_%s_COMPILER_RT_BUILD_SANITIZERS=OFF" % target,
+                "-DRUNTIMES_%s_COMPILER_RT_INCLUDE_TESTS=OFF" % target,
+                "-DRUNTIMES_%s_LLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF" % target,
+                "-DRUNTIMES_%s_LLVM_INCLUDE_TESTS=OFF" % target,
+                "-DRUNTIMES_%s_ANDROID_NATIVE_API_LEVEL=16" % target,
+            ]
+
+    cmake_args += cmake_base_args(
         cc, cxx, asm, ld, ar, ranlib, libtool, inst_dir)
     cmake_args += [
         src_dir
     ]
     build_package(build_dir, cmake_args)
 
     if is_linux():
         install_libgcc(gcc_dir, inst_dir)
@@ -529,16 +577,25 @@ if __name__ == "__main__":
     if "python_path" not in config:
         raise ValueError("Config file needs to set python_path")
     python_path = config["python_path"]
     gcc_dir = None
     if "gcc_dir" in config:
         gcc_dir = config["gcc_dir"]
         if not os.path.exists(gcc_dir):
             raise ValueError("gcc_dir must point to an existing path")
+    ndk_dir = None
+    android_targets = None
+    if "android_targets" in config:
+        android_targets = config["android_targets"]
+        for attr in ("ndk_toolchain", "ndk_sysroot", "ndk_includes"):
+            for target, cfg in android_targets.iteritems():
+                if attr not in cfg:
+                    raise ValueError("must specify '%s' as a key for android target: %s" %
+                                     (attr, target))
     if is_linux() and gcc_dir is None:
         raise ValueError("Config file needs to set gcc_dir")
     cc = get_tool(config, "cc")
     cxx = get_tool(config, "cxx")
     asm = get_tool(config, "ml" if is_windows() else "as")
     ld = get_tool(config, "link" if is_windows() else "ld")
     ar = get_tool(config, "lib" if is_windows() else "ar")
     ranlib = None if is_windows() else get_tool(config, "ranlib")
@@ -565,24 +622,25 @@ if __name__ == "__main__":
             checkout_or_update(lld_repo, lld_source_dir)
         if libcxxabi_repo:
             checkout_or_update(libcxxabi_repo, libcxxabi_source_dir)
         if extra_repo:
             checkout_or_update(extra_repo, extra_source_dir)
         for p in config.get("patches", []):
             patch(p, source_dir)
 
+    compiler_rt_source_link = llvm_source_dir + "/projects/compiler-rt"
+
     symlinks = [(clang_source_dir,
                  llvm_source_dir + "/tools/clang"),
                 (extra_source_dir,
                  llvm_source_dir + "/tools/clang/tools/extra"),
                 (lld_source_dir,
                  llvm_source_dir + "/tools/lld"),
-                (compiler_rt_source_dir,
-                 llvm_source_dir + "/projects/compiler-rt"),
+                (compiler_rt_source_dir, compiler_rt_source_link),
                 (libcxx_source_dir,
                  llvm_source_dir + "/projects/libcxx"),
                 (libcxxabi_source_dir,
                  llvm_source_dir + "/projects/libcxxabi")]
     for l in symlinks:
         # On Windows, we have to re-copy the whole directory every time.
         if not is_windows() and os.path.islink(l[1]):
             continue
@@ -665,48 +723,57 @@ if __name__ == "__main__":
         [cc] + extra_cflags,
         [cxx] + extra_cxxflags,
         [asm] + extra_asmflags,
         [ld] + extra_ldflags,
         ar, ranlib, libtool,
         llvm_source_dir, stage1_dir, package_name, build_libcxx, osx_cross_compile,
         build_type, assertions, python_path, gcc_dir, libcxx_include_dir)
 
+    if android_targets:
+        # Linking compiler-rt under "runtimes" activates LLVM_RUNTIME_TARGETS
+        # and related arguments.
+        symlink(compiler_rt_source_dir, llvm_source_dir + "/runtimes/compiler-rt")
+        try:
+            os.unlink(compiler_rt_source_link)
+        except Exception:
+            pass
+
     if stages > 1:
         stage2_dir = build_dir + '/stage2'
         stage2_inst_dir = stage2_dir + '/' + package_name
         final_stage_dir = stage2_dir
         build_one_stage(
             [stage1_inst_dir + "/bin/%s%s" %
                 (cc_name, exe_ext)] + extra_cflags2,
             [stage1_inst_dir + "/bin/%s%s" %
                 (cxx_name, exe_ext)] + extra_cxxflags2,
             [stage1_inst_dir + "/bin/%s%s" %
                 (cc_name, exe_ext)] + extra_asmflags,
             [ld] + extra_ldflags,
             ar, ranlib, libtool,
             llvm_source_dir, stage2_dir, package_name, build_libcxx, osx_cross_compile,
             build_type, assertions, python_path, gcc_dir, libcxx_include_dir,
-            stages == 2)
+            is_final_stage=(stages == 2), android_targets=android_targets)
 
     if stages > 2:
         stage3_dir = build_dir + '/stage3'
         final_stage_dir = stage3_dir
         build_one_stage(
             [stage2_inst_dir + "/bin/%s%s" %
                 (cc_name, exe_ext)] + extra_cflags2,
             [stage2_inst_dir + "/bin/%s%s" %
                 (cxx_name, exe_ext)] + extra_cxxflags2,
             [stage2_inst_dir + "/bin/%s%s" %
                 (cc_name, exe_ext)] + extra_asmflags,
             [ld] + extra_ldflags,
             ar, ranlib, libtool,
             llvm_source_dir, stage3_dir, package_name, build_libcxx, osx_cross_compile,
             build_type, assertions, python_path, gcc_dir, libcxx_include_dir,
-            stages == 3)
+            (stages == 3))
 
     if build_clang_tidy:
         prune_final_dir_for_clang_tidy(os.path.join(final_stage_dir, package_name),
                                        osx_cross_compile)
 
     if not args.skip_tar:
         ext = "bz2" if is_darwin() or is_windows() else "xz"
         build_tar_package("tar", "%s.tar.%s" % (package_name, ext), final_stage_dir, package_name)
new file mode 100644
--- /dev/null
+++ b/build/build-clang/clang-7-android.json
@@ -0,0 +1,33 @@
+{
+    "llvm_revision": "342383",
+    "stages": "2",
+    "build_libcxx": true,
+    "build_type": "Release",
+    "assertions": false,
+    "llvm_repo": "https://llvm.org/svn/llvm-project/llvm/tags/RELEASE_700/final",
+    "clang_repo": "https://llvm.org/svn/llvm-project/cfe/tags/RELEASE_700/final",
+    "lld_repo": "https://llvm.org/svn/llvm-project/lld/tags/RELEASE_700/final",
+    "compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/tags/RELEASE_700/final",
+    "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/tags/RELEASE_700/final",
+    "libcxxabi_repo": "https://llvm.org/svn/llvm-project/libcxxabi/tags/RELEASE_700/final",
+    "python_path": "/usr/bin/python2.7",
+    "gcc_dir": "/builds/worker/workspace/build/src/gcc",
+    "cc": "/builds/worker/workspace/build/src/gcc/bin/gcc",
+    "cxx": "/builds/worker/workspace/build/src/gcc/bin/g++",
+    "as": "/builds/worker/workspace/build/src/gcc/bin/gcc",
+    "android_targets": {
+      "armv7-linux-android": {
+        "ndk_toolchain": "/builds/worker/workspace/build/src/android-ndk/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64",
+        "ndk_sysroot": "/builds/worker/workspace/build/src/android-ndk/platforms/android-16/arch-arm",
+        "ndk_includes": [
+          "/builds/worker/workspace/build/src/android-ndk/sysroot/usr/include/arm-linux-androideabi",
+          "/builds/worker/workspace/build/src/android-ndk/sysroot/usr/include"
+        ]
+      }
+    },
+    "patches": [
+      "static-llvm-symbolizer.patch",
+      "find_symbolizer_linux.patch",
+      "rename_gcov_flush_.patch"
+    ]
+}
--- a/taskcluster/ci/build/android.yml
+++ b/taskcluster/ci/build/android.yml
@@ -317,17 +317,17 @@ android-api-16/opt:
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         custom-build-variant-cfg: api-16
         tooltool-downloads: internal
     toolchains:
         - android-gradle-dependencies
         - android-ndk-linux
         - android-sdk-linux
-        - linux64-clang
+        - linux64-clang-android-cross
         - linux64-rust-android
         - linux64-rust-size
         - linux64-cbindgen
         - linux64-sccache
         - linux64-node
 
 android-api-16-without-google-play-services/opt:
     description: "Android 4.0 api-16+ (without Google Play Services) Opt"
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@@ -92,16 +92,39 @@ linux64-clang-trunk-mingw-x64:
         resources:
             - 'build/build-clang/build-clang.py'
             - 'build/build-clang/clang-trunk-mingw.json'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/clangmingw.tar.xz
     toolchains:
         - linux64-gcc-4.9
 
+linux64-clang-7-android-cross:
+    description: "Clang 7 toolchain build"
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TL(clang7-android)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux-xlarge
+    worker:
+        max-run-time: 7200
+    run:
+        using: toolchain-script
+        script: build-clang-7-android.sh
+        resources:
+            - 'build/build-clang/build-clang.py'
+            - 'build/build-clang/clang-7-android.json'
+            - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-alias: linux64-clang-android-cross
+        toolchain-artifact: public/build/clang.tar.xz
+    toolchains:
+        - linux64-gcc-4.9
+        - linux64-android-ndk-linux-repack
+
 linux64-clang-7-macosx-cross:
     description: "Clang 7 toolchain build with MacOS Compiler RT libs"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(clang7-macosx-cross)
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/misc/build-clang-7-android.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+set -x -e -v
+
+# This script is for building clang on a Linux host with android compiler rt
+# libs.
+
+WORKSPACE=$HOME/workspace
+HOME_DIR=$WORKSPACE/build
+UPLOAD_DIR=$HOME/artifacts
+
+cd $HOME_DIR/src
+
+. taskcluster/scripts/misc/tooltool-download.sh
+
+# gets a bit too verbose here
+set +x
+
+cd build/build-clang
+# |mach python| sets up a virtualenv for us!
+../../mach python ./build-clang.py -c clang-7-android.json
+
+set -x
+
+# Put a tarball in the artifacts dir
+mkdir -p $UPLOAD_DIR
+cp clang.tar.* $UPLOAD_DIR