Bug 1479800 - Build 32-bits compiler-rt for 64-bits clang-cl. r=dmajor
☠☠ backed out by 8adbf514581c ☠ ☠
authorMike Hommey <mh+mozilla@glandium.org>
Thu, 16 Aug 2018 10:31:03 +0900
changeset 487179 dccd5299c5ad9e4258eb7b08201f2f39164a2a54
parent 487178 16daef3a89a23e7805469b25c21cc31f5dade193
child 487180 5fcbe08fb321a5076f56c380b554c63068efbb00
push id9719
push userffxbld-merge
push dateFri, 24 Aug 2018 17:49:46 +0000
treeherdermozilla-beta@719ec98fba77 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdmajor
bugs1479800
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1479800 - Build 32-bits compiler-rt for 64-bits clang-cl. r=dmajor
build/build-clang/build-clang.py
build/build-clang/clang-win64-st-an.json
build/build-clang/clang-win64.json
build/build-clang/r327876.patch
build/build-clang/workaround-issue38586.patch
--- a/build/build-clang/build-clang.py
+++ b/build/build-clang/build-clang.py
@@ -58,16 +58,24 @@ def import_clang_tidy(source_dir):
     sys.path.append(clang_plugin_path)
     from import_mozilla_checks import do_import
     do_import(clang_plugin_path, clang_tidy_path)
 
 
 def build_package(package_build_dir, cmake_args):
     if not os.path.exists(package_build_dir):
         os.mkdir(package_build_dir)
+    # If CMake has already been run, it may have been run with different
+    # arguments, so we need to re-run it.  Make sure the cached copy of the
+    # previous CMake run is cleared before running it again.
+    if os.path.exists(package_build_dir + "/CMakeCache.txt"):
+        os.remove(package_build_dir + "/CMakeCache.txt")
+    if os.path.exists(package_build_dir + "/CMakeFiles"):
+        shutil.rmtree(package_build_dir + "/CMakeFiles")
+
     run_in(package_build_dir, ["cmake"] + cmake_args)
     run_in(package_build_dir, ["ninja", "install"])
 
 
 @contextmanager
 def updated_env(env):
     old_env = os.environ.copy()
     os.environ.update(env)
@@ -171,83 +179,125 @@ def is_linux():
 
 def is_windows():
     return platform.system() == "Windows"
 
 
 def build_one_stage(cc, cxx, asm, ld, ar, ranlib, libtool,
                     src_dir, stage_dir, build_libcxx,
                     osx_cross_compile, build_type, assertions,
-                    python_path, gcc_dir, libcxx_include_dir):
+                    python_path, gcc_dir, libcxx_include_dir,
+                    is_final_stage=False):
     if not os.path.exists(stage_dir):
         os.mkdir(stage_dir)
 
     build_dir = stage_dir + "/build"
     inst_dir = stage_dir + "/clang"
 
-    # If CMake has already been run, it may have been run with different
-    # arguments, so we need to re-run it.  Make sure the cached copy of the
-    # previous CMake run is cleared before running it again.
-    if os.path.exists(build_dir + "/CMakeCache.txt"):
-        os.remove(build_dir + "/CMakeCache.txt")
-    if os.path.exists(build_dir + "/CMakeFiles"):
-        shutil.rmtree(build_dir + "/CMakeFiles")
-
     # cmake doesn't deal well with backslashes in paths.
     def slashify_path(path):
         return path.replace('\\', '/')
 
-    cmake_args = ["-GNinja",
-                  "-DCMAKE_C_COMPILER=%s" % slashify_path(cc[0]),
-                  "-DCMAKE_CXX_COMPILER=%s" % slashify_path(cxx[0]),
-                  "-DCMAKE_ASM_COMPILER=%s" % slashify_path(asm[0]),
-                  "-DCMAKE_LINKER=%s" % slashify_path(ld[0]),
-                  "-DCMAKE_AR=%s" % slashify_path(ar),
-                  "-DCMAKE_C_FLAGS=%s" % ' '.join(cc[1:]),
-                  "-DCMAKE_CXX_FLAGS=%s" % ' '.join(cxx[1:]),
-                  "-DCMAKE_ASM_FLAGS=%s" % ' '.join(asm[1:]),
-                  "-DCMAKE_EXE_LINKER_FLAGS=%s" % ' '.join(ld[1:]),
-                  "-DCMAKE_SHARED_LINKER_FLAGS=%s" % ' '.join(ld[1:]),
-                  "-DCMAKE_BUILD_TYPE=%s" % build_type,
-                  "-DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64",
-                  "-DLLVM_ENABLE_ASSERTIONS=%s" % ("ON" if assertions else "OFF"),
-                  "-DPYTHON_EXECUTABLE=%s" % slashify_path(python_path),
-                  "-DCMAKE_INSTALL_PREFIX=%s" % inst_dir,
-                  "-DLLVM_TOOL_LIBCXX_BUILD=%s" % ("ON" if build_libcxx else "OFF"),
-                  "-DLIBCXX_LIBCPPABI_VERSION=\"\"",
-                  src_dir]
-    if is_windows():
-        cmake_args.insert(-1, "-DLLVM_EXPORT_SYMBOLS_FOR_PLUGINS=ON")
-        cmake_args.insert(-1, "-DLLVM_USE_CRT_RELEASE=MT")
-    if ranlib is not None:
-        cmake_args += ["-DCMAKE_RANLIB=%s" % slashify_path(ranlib)]
-    if libtool is not None:
-        cmake_args += ["-DCMAKE_LIBTOOL=%s" % slashify_path(libtool)]
-    if osx_cross_compile:
-        cmake_args += ["-DCMAKE_SYSTEM_NAME=Darwin",
-                       "-DCMAKE_SYSTEM_VERSION=10.10",
-                       "-DLLVM_ENABLE_THREADS=OFF",
-                       "-DLIBCXXABI_LIBCXX_INCLUDES=%s" % libcxx_include_dir,
-                       "-DCMAKE_OSX_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
-                       "-DCMAKE_FIND_ROOT_PATH=%s" % slashify_path(os.getenv("CROSS_CCTOOLS_PATH")), # noqa
-                       "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER",
-                       "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY",
-                       "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY",
-                       "-DCMAKE_MACOSX_RPATH=ON",
-                       "-DCMAKE_OSX_ARCHITECTURES=x86_64",
-                       "-DDARWIN_osx_ARCHS=x86_64",
-                       "-DDARWIN_osx_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
-                       "-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-apple-darwin11"]
+    def cmake_base_args(cc, cxx, asm, ld, ar, ranlib, libtool, inst_dir):
+        cmake_args = [
+            "-GNinja",
+            "-DCMAKE_C_COMPILER=%s" % slashify_path(cc[0]),
+            "-DCMAKE_CXX_COMPILER=%s" % slashify_path(cxx[0]),
+            "-DCMAKE_ASM_COMPILER=%s" % slashify_path(asm[0]),
+            "-DCMAKE_LINKER=%s" % slashify_path(ld[0]),
+            "-DCMAKE_AR=%s" % slashify_path(ar),
+            "-DCMAKE_C_FLAGS=%s" % ' '.join(cc[1:]),
+            "-DCMAKE_CXX_FLAGS=%s" % ' '.join(cxx[1:]),
+            "-DCMAKE_ASM_FLAGS=%s" % ' '.join(asm[1:]),
+            "-DCMAKE_EXE_LINKER_FLAGS=%s" % ' '.join(ld[1:]),
+            "-DCMAKE_SHARED_LINKER_FLAGS=%s" % ' '.join(ld[1:]),
+            "-DCMAKE_BUILD_TYPE=%s" % build_type,
+            "-DCMAKE_INSTALL_PREFIX=%s" % inst_dir,
+            "-DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64",
+            "-DLLVM_ENABLE_ASSERTIONS=%s" % ("ON" if assertions else "OFF"),
+            "-DPYTHON_EXECUTABLE=%s" % slashify_path(python_path),
+            "-DLLVM_TOOL_LIBCXX_BUILD=%s" % ("ON" if build_libcxx else "OFF"),
+            "-DLIBCXX_LIBCPPABI_VERSION=\"\"",
+        ]
+        if is_windows():
+            cmake_args.insert(-1, "-DLLVM_EXPORT_SYMBOLS_FOR_PLUGINS=ON")
+            cmake_args.insert(-1, "-DLLVM_USE_CRT_RELEASE=MT")
+        if ranlib is not None:
+            cmake_args += ["-DCMAKE_RANLIB=%s" % slashify_path(ranlib)]
+        if libtool is not None:
+            cmake_args += ["-DCMAKE_LIBTOOL=%s" % slashify_path(libtool)]
+        if osx_cross_compile:
+            cmake_args += [
+                "-DCMAKE_SYSTEM_NAME=Darwin",
+                "-DCMAKE_SYSTEM_VERSION=10.10",
+                "-DLLVM_ENABLE_THREADS=OFF",
+                "-DLIBCXXABI_LIBCXX_INCLUDES=%s" % libcxx_include_dir,
+                "-DCMAKE_OSX_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
+                "-DCMAKE_FIND_ROOT_PATH=%s" % slashify_path(os.getenv("CROSS_CCTOOLS_PATH")), # noqa
+                "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER",
+                "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY",
+                "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY",
+                "-DCMAKE_MACOSX_RPATH=ON",
+                "-DCMAKE_OSX_ARCHITECTURES=x86_64",
+                "-DDARWIN_osx_ARCHS=x86_64",
+                "-DDARWIN_osx_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
+                "-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-apple-darwin11"
+            ]
+        return cmake_args
+
+    cmake_args = cmake_base_args(
+        cc, cxx, asm, ld, ar, ranlib, libtool, inst_dir)
+    cmake_args += [
+        src_dir
+    ]
     build_package(build_dir, cmake_args)
 
     if is_linux():
         install_libgcc(gcc_dir, inst_dir)
     # For some reasons the import library clang.lib of clang.exe is not
     # installed, so we copy it by ourselves.
     if is_windows():
+        # The compiler-rt cmake scripts don't allow to build it for multiple
+        # targets at once on Windows, so manually build the 32-bits compiler-rt
+        # during the final stage.
+        build_32_bit = False
+        if is_final_stage:
+            # Only build the 32-bits compiler-rt when we originally built for
+            # 64-bits, which we detect through the contents of the LIB
+            # environment variable, which we also adjust for a 32-bits build
+            # at the same time.
+            old_lib = os.environ['LIB']
+            new_lib = []
+            for l in old_lib.split(os.pathsep):
+                if l.endswith('x64'):
+                    l = l[:-3] + 'x86'
+                    build_32_bit = True
+                elif l.endswith('amd64'):
+                    l = l[:-5]
+                    build_32_bit = True
+                new_lib.append(l)
+        if build_32_bit:
+            os.environ['LIB'] = os.pathsep.join(new_lib)
+            compiler_rt_build_dir = stage_dir + '/compiler-rt'
+            compiler_rt_inst_dir = inst_dir + '/lib/clang/'
+            subdirs = os.listdir(compiler_rt_inst_dir)
+            assert len(subdirs) == 1
+            compiler_rt_inst_dir += subdirs[0]
+            cmake_args = cmake_base_args(
+                [os.path.join(inst_dir, 'bin', 'clang-cl.exe'), '-m32'] + cc[1:],
+                [os.path.join(inst_dir, 'bin', 'clang-cl.exe'), '-m32'] + cxx[1:],
+                [os.path.join(inst_dir, 'bin', 'clang-cl.exe'), '-m32'] + asm[1:],
+                ld, ar, ranlib, libtool, compiler_rt_inst_dir)
+            cmake_args += [
+                '-DLLVM_CONFIG_PATH=%s' % slashify_path(
+                    os.path.join(inst_dir, 'bin', 'llvm-config')),
+                os.path.join(src_dir, 'projects', 'compiler-rt'),
+            ]
+            build_package(compiler_rt_build_dir, cmake_args)
+            os.environ['LIB'] = old_lib
         install_import_library(build_dir, inst_dir)
         install_asan_symbols(build_dir, inst_dir)
 
 
 # Return the absolute path of a build tool.  We first look to see if the
 # variable is defined in the config file, and if so we make sure it's an
 # absolute path to an existing tool, otherwise we look for a program in
 # $PATH named "key".
@@ -610,32 +660,34 @@ if __name__ == "__main__":
                 (cc_name, exe_ext)] + extra_cflags2,
             [stage1_inst_dir + "/bin/%s%s" %
                 (cxx_name, exe_ext)] + extra_cxxflags2,
             [stage1_inst_dir + "/bin/%s%s" %
                 (cc_name, exe_ext)] + extra_asmflags,
             [ld] + extra_ldflags,
             ar, ranlib, libtool,
             llvm_source_dir, stage2_dir, build_libcxx, osx_cross_compile,
-            build_type, assertions, python_path, gcc_dir, libcxx_include_dir)
+            build_type, assertions, python_path, gcc_dir, libcxx_include_dir,
+            stages == 2)
 
     if stages > 2:
         stage3_dir = build_dir + '/stage3'
         final_stage_dir = stage3_dir
         build_one_stage(
             [stage2_inst_dir + "/bin/%s%s" %
                 (cc_name, exe_ext)] + extra_cflags2,
             [stage2_inst_dir + "/bin/%s%s" %
                 (cxx_name, exe_ext)] + extra_cxxflags2,
             [stage2_inst_dir + "/bin/%s%s" %
                 (cc_name, exe_ext)] + extra_asmflags,
             [ld] + extra_ldflags,
             ar, ranlib, libtool,
             llvm_source_dir, stage3_dir, build_libcxx, osx_cross_compile,
-            build_type, assertions, python_path, gcc_dir, libcxx_include_dir)
+            build_type, assertions, python_path, gcc_dir, libcxx_include_dir,
+            stages == 3)
 
     package_name = "clang"
     if build_clang_tidy:
         prune_final_dir_for_clang_tidy(os.path.join(final_stage_dir, "clang"))
         package_name = "clang-tidy"
 
     if not args.skip_tar:
         ext = "bz2" if is_darwin() or is_windows() else "xz"
--- a/build/build-clang/clang-win64-st-an.json
+++ b/build/build-clang/clang-win64-st-an.json
@@ -11,12 +11,13 @@
     "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
     "python_path": "c:/mozilla-build/python/python.exe",
     "cc": "cl.exe",
     "cxx": "cl.exe",
     "ml": "ml64.exe",
     "patches": [
       "r318309.patch",
       "r320462.patch",
+      "r327876.patch",
       "loosen-msvc-detection.patch",
       "fflush-before-unlocking.patch"
     ]
 }
--- a/build/build-clang/clang-win64.json
+++ b/build/build-clang/clang-win64.json
@@ -10,11 +10,12 @@
     "compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/trunk",
     "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
     "python_path": "c:/mozilla-build/python/python.exe",
     "cc": "cl.exe",
     "cxx": "cl.exe",
     "ml": "ml64.exe",
     "patches": [
       "loosen-msvc-detection.patch",
-      "r339636.patch"
+      "r339636.patch",
+      "workaround-issue38586.patch"
     ]
 }
new file mode 100644
--- /dev/null
+++ b/build/build-clang/r327876.patch
@@ -0,0 +1,39 @@
+From 522a892efc2ff22a2fd421b1ef4d9d9739d78b2e Mon Sep 17 00:00:00 2001
+From: Vitaly Buka <vitalybuka@google.com>
+Date: Mon, 19 Mar 2018 18:22:35 +0000
+Subject: [PATCH] Fix CMake/MSVC when compiler-rt and llvm are built separately
+
+Summary:
+For some reason CMake can't find the `append` macro if LLVM is built separately and imported via `LLVM_CONFIG_PATH`.
+
+Patch by Loo Rong Jie
+
+Reviewers: rnk, vitalybuka
+
+Reviewed By: rnk, vitalybuka
+
+Subscribers: dberris, mgorny, llvm-commits, #sanitizers
+
+Differential Revision: https://reviews.llvm.org/D43458
+
+git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@327876 91177308-0d34-0410-b5e6-96231b3b80d8
+---
+ CMakeLists.txt | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt
+index 4b953b212..a02ef9532 100644
+--- a/compiler-rt/CMakeLists.txt
++++ b/compiler-rt/CMakeLists.txt
+@@ -339,7 +339,7 @@ if (CMAKE_LINKER MATCHES "link.exe$")
+   # it, but CMake doesn't seem to have a way to set linker flags for
+   # individual static libraries, so we enable the suppression flag for
+   # the whole compiler-rt project.
+-  append("/IGNORE:4221" CMAKE_STATIC_LINKER_FLAGS)
++  set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} /IGNORE:4221")
+ endif()
+ 
+ add_subdirectory(include)
+-- 
+2.18.0
+
new file mode 100644
--- /dev/null
+++ b/build/build-clang/workaround-issue38586.patch
@@ -0,0 +1,31 @@
+diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt
+index 86ca2b3ef..f6ddd24eb 100644
+--- a/compiler-rt/CMakeLists.txt
++++ b/compiler-rt/CMakeLists.txt
+@@ -284,6 +284,26 @@ if(MSVC)
+   # warning from the MS linker complaining that it can't find the 'vc140.pdb'
+   # file used by our object library compilations.
+   list(APPEND SANITIZER_COMMON_CFLAGS /Z7)
++
++# Copied from llvm/cmake/modules/LLVMProcessSources.cmake
++function(llvm_replace_compiler_option var old new)
++  # Replaces a compiler option or switch `old' in `var' by `new'.
++  # If `old' is not in `var', appends `new' to `var'.
++  # Example: llvm_replace_compiler_option(CMAKE_CXX_FLAGS_RELEASE "-O3" "-O2")
++  # If the option already is on the variable, don't add it:
++  if( "${${var}}" MATCHES "(^| )${new}($| )" )
++    set(n "")
++  else()
++    set(n "${new}")
++  endif()
++  if( "${${var}}" MATCHES "(^| )${old}($| )" )
++    string( REGEX REPLACE "(^| )${old}($| )" " ${n} " ${var} "${${var}}" )
++  else()
++    set( ${var} "${${var}} ${n}" )
++  endif()
++  set( ${var} "${${var}}" PARENT_SCOPE )
++endfunction(llvm_replace_compiler_option)
++
+   llvm_replace_compiler_option(CMAKE_CXX_FLAGS "/Z[i7I]" "/Z7")
+   llvm_replace_compiler_option(CMAKE_CXX_FLAGS_DEBUG "/Z[i7I]" "/Z7")
+   llvm_replace_compiler_option(CMAKE_CXX_FLAGS_RELWITHDEBINFO "/Z[i7I]" "/Z7")