Bug 1479800 - Build 32-bits compiler-rt for 64-bits clang-cl. r=dmajor
☠☠ backed out by 8adbf514581c ☠ ☠
authorMike Hommey <mh+mozilla@glandium.org>
Thu, 16 Aug 2018 10:31:03 +0900
changeset 829894 dccd5299c5ad9e4258eb7b08201f2f39164a2a54
parent 829893 16daef3a89a23e7805469b25c21cc31f5dade193
child 829895 5fcbe08fb321a5076f56c380b554c63068efbb00
push id118801
push userbmo:rbartlensky@mozilla.com
push dateFri, 17 Aug 2018 12:50:32 +0000
reviewersdmajor
bugs1479800
milestone63.0a1
Bug 1479800 - Build 32-bits compiler-rt for 64-bits clang-cl. r=dmajor
build/build-clang/build-clang.py
build/build-clang/clang-win64-st-an.json
build/build-clang/clang-win64.json
build/build-clang/r327876.patch
build/build-clang/workaround-issue38586.patch
--- a/build/build-clang/build-clang.py
+++ b/build/build-clang/build-clang.py
@@ -58,16 +58,24 @@ def import_clang_tidy(source_dir):
     sys.path.append(clang_plugin_path)
     from import_mozilla_checks import do_import
     do_import(clang_plugin_path, clang_tidy_path)
 
 
 def build_package(package_build_dir, cmake_args):
     if not os.path.exists(package_build_dir):
         os.mkdir(package_build_dir)
+    # If CMake has already been run, it may have been run with different
+    # arguments, so we need to re-run it.  Make sure the cached copy of the
+    # previous CMake run is cleared before running it again.
+    if os.path.exists(package_build_dir + "/CMakeCache.txt"):
+        os.remove(package_build_dir + "/CMakeCache.txt")
+    if os.path.exists(package_build_dir + "/CMakeFiles"):
+        shutil.rmtree(package_build_dir + "/CMakeFiles")
+
     run_in(package_build_dir, ["cmake"] + cmake_args)
     run_in(package_build_dir, ["ninja", "install"])
 
 
 @contextmanager
 def updated_env(env):
     old_env = os.environ.copy()
     os.environ.update(env)
@@ -171,83 +179,125 @@ def is_linux():
 
 def is_windows():
     return platform.system() == "Windows"
 
 
 def build_one_stage(cc, cxx, asm, ld, ar, ranlib, libtool,
                     src_dir, stage_dir, build_libcxx,
                     osx_cross_compile, build_type, assertions,
-                    python_path, gcc_dir, libcxx_include_dir):
+                    python_path, gcc_dir, libcxx_include_dir,
+                    is_final_stage=False):
     if not os.path.exists(stage_dir):
         os.mkdir(stage_dir)
 
     build_dir = stage_dir + "/build"
     inst_dir = stage_dir + "/clang"
 
-    # If CMake has already been run, it may have been run with different
-    # arguments, so we need to re-run it.  Make sure the cached copy of the
-    # previous CMake run is cleared before running it again.
-    if os.path.exists(build_dir + "/CMakeCache.txt"):
-        os.remove(build_dir + "/CMakeCache.txt")
-    if os.path.exists(build_dir + "/CMakeFiles"):
-        shutil.rmtree(build_dir + "/CMakeFiles")
-
     # cmake doesn't deal well with backslashes in paths.
     def slashify_path(path):
         return path.replace('\\', '/')
 
-    cmake_args = ["-GNinja",
-                  "-DCMAKE_C_COMPILER=%s" % slashify_path(cc[0]),
-                  "-DCMAKE_CXX_COMPILER=%s" % slashify_path(cxx[0]),
-                  "-DCMAKE_ASM_COMPILER=%s" % slashify_path(asm[0]),
-                  "-DCMAKE_LINKER=%s" % slashify_path(ld[0]),
-                  "-DCMAKE_AR=%s" % slashify_path(ar),
-                  "-DCMAKE_C_FLAGS=%s" % ' '.join(cc[1:]),
-                  "-DCMAKE_CXX_FLAGS=%s" % ' '.join(cxx[1:]),
-                  "-DCMAKE_ASM_FLAGS=%s" % ' '.join(asm[1:]),
-                  "-DCMAKE_EXE_LINKER_FLAGS=%s" % ' '.join(ld[1:]),
-                  "-DCMAKE_SHARED_LINKER_FLAGS=%s" % ' '.join(ld[1:]),
-                  "-DCMAKE_BUILD_TYPE=%s" % build_type,
-                  "-DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64",
-                  "-DLLVM_ENABLE_ASSERTIONS=%s" % ("ON" if assertions else "OFF"),
-                  "-DPYTHON_EXECUTABLE=%s" % slashify_path(python_path),
-                  "-DCMAKE_INSTALL_PREFIX=%s" % inst_dir,
-                  "-DLLVM_TOOL_LIBCXX_BUILD=%s" % ("ON" if build_libcxx else "OFF"),
-                  "-DLIBCXX_LIBCPPABI_VERSION=\"\"",
-                  src_dir]
-    if is_windows():
-        cmake_args.insert(-1, "-DLLVM_EXPORT_SYMBOLS_FOR_PLUGINS=ON")
-        cmake_args.insert(-1, "-DLLVM_USE_CRT_RELEASE=MT")
-    if ranlib is not None:
-        cmake_args += ["-DCMAKE_RANLIB=%s" % slashify_path(ranlib)]
-    if libtool is not None:
-        cmake_args += ["-DCMAKE_LIBTOOL=%s" % slashify_path(libtool)]
-    if osx_cross_compile:
-        cmake_args += ["-DCMAKE_SYSTEM_NAME=Darwin",
-                       "-DCMAKE_SYSTEM_VERSION=10.10",
-                       "-DLLVM_ENABLE_THREADS=OFF",
-                       "-DLIBCXXABI_LIBCXX_INCLUDES=%s" % libcxx_include_dir,
-                       "-DCMAKE_OSX_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
-                       "-DCMAKE_FIND_ROOT_PATH=%s" % slashify_path(os.getenv("CROSS_CCTOOLS_PATH")), # noqa
-                       "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER",
-                       "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY",
-                       "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY",
-                       "-DCMAKE_MACOSX_RPATH=ON",
-                       "-DCMAKE_OSX_ARCHITECTURES=x86_64",
-                       "-DDARWIN_osx_ARCHS=x86_64",
-                       "-DDARWIN_osx_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
-                       "-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-apple-darwin11"]
+    def cmake_base_args(cc, cxx, asm, ld, ar, ranlib, libtool, inst_dir):
+        cmake_args = [
+            "-GNinja",
+            "-DCMAKE_C_COMPILER=%s" % slashify_path(cc[0]),
+            "-DCMAKE_CXX_COMPILER=%s" % slashify_path(cxx[0]),
+            "-DCMAKE_ASM_COMPILER=%s" % slashify_path(asm[0]),
+            "-DCMAKE_LINKER=%s" % slashify_path(ld[0]),
+            "-DCMAKE_AR=%s" % slashify_path(ar),
+            "-DCMAKE_C_FLAGS=%s" % ' '.join(cc[1:]),
+            "-DCMAKE_CXX_FLAGS=%s" % ' '.join(cxx[1:]),
+            "-DCMAKE_ASM_FLAGS=%s" % ' '.join(asm[1:]),
+            "-DCMAKE_EXE_LINKER_FLAGS=%s" % ' '.join(ld[1:]),
+            "-DCMAKE_SHARED_LINKER_FLAGS=%s" % ' '.join(ld[1:]),
+            "-DCMAKE_BUILD_TYPE=%s" % build_type,
+            "-DCMAKE_INSTALL_PREFIX=%s" % inst_dir,
+            "-DLLVM_TARGETS_TO_BUILD=X86;ARM;AArch64",
+            "-DLLVM_ENABLE_ASSERTIONS=%s" % ("ON" if assertions else "OFF"),
+            "-DPYTHON_EXECUTABLE=%s" % slashify_path(python_path),
+            "-DLLVM_TOOL_LIBCXX_BUILD=%s" % ("ON" if build_libcxx else "OFF"),
+            "-DLIBCXX_LIBCPPABI_VERSION=\"\"",
+        ]
+        if is_windows():
+            cmake_args.insert(-1, "-DLLVM_EXPORT_SYMBOLS_FOR_PLUGINS=ON")
+            cmake_args.insert(-1, "-DLLVM_USE_CRT_RELEASE=MT")
+        if ranlib is not None:
+            cmake_args += ["-DCMAKE_RANLIB=%s" % slashify_path(ranlib)]
+        if libtool is not None:
+            cmake_args += ["-DCMAKE_LIBTOOL=%s" % slashify_path(libtool)]
+        if osx_cross_compile:
+            cmake_args += [
+                "-DCMAKE_SYSTEM_NAME=Darwin",
+                "-DCMAKE_SYSTEM_VERSION=10.10",
+                "-DLLVM_ENABLE_THREADS=OFF",
+                "-DLIBCXXABI_LIBCXX_INCLUDES=%s" % libcxx_include_dir,
+                "-DCMAKE_OSX_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
+                "-DCMAKE_FIND_ROOT_PATH=%s" % slashify_path(os.getenv("CROSS_CCTOOLS_PATH")), # noqa
+                "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER",
+                "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY",
+                "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY",
+                "-DCMAKE_MACOSX_RPATH=ON",
+                "-DCMAKE_OSX_ARCHITECTURES=x86_64",
+                "-DDARWIN_osx_ARCHS=x86_64",
+                "-DDARWIN_osx_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
+                "-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-apple-darwin11"
+            ]
+        return cmake_args
+
+    cmake_args = cmake_base_args(
+        cc, cxx, asm, ld, ar, ranlib, libtool, inst_dir)
+    cmake_args += [
+        src_dir
+    ]
     build_package(build_dir, cmake_args)
 
     if is_linux():
         install_libgcc(gcc_dir, inst_dir)
     # For some reasons the import library clang.lib of clang.exe is not
     # installed, so we copy it by ourselves.
     if is_windows():
+        # The compiler-rt cmake scripts don't allow to build it for multiple
+        # targets at once on Windows, so manually build the 32-bits compiler-rt
+        # during the final stage.
+        build_32_bit = False
+        if is_final_stage:
+            # Only build the 32-bits compiler-rt when we originally built for
+            # 64-bits, which we detect through the contents of the LIB
+            # environment variable, which we also adjust for a 32-bits build
+            # at the same time.
+            old_lib = os.environ['LIB']
+            new_lib = []
+            for l in old_lib.split(os.pathsep):
+                if l.endswith('x64'):
+                    l = l[:-3] + 'x86'
+                    build_32_bit = True
+                elif l.endswith('amd64'):
+                    l = l[:-5]
+                    build_32_bit = True
+                new_lib.append(l)
+        if build_32_bit:
+            os.environ['LIB'] = os.pathsep.join(new_lib)
+            compiler_rt_build_dir = stage_dir + '/compiler-rt'
+            compiler_rt_inst_dir = inst_dir + '/lib/clang/'
+            subdirs = os.listdir(compiler_rt_inst_dir)
+            assert len(subdirs) == 1
+            compiler_rt_inst_dir += subdirs[0]
+            cmake_args = cmake_base_args(
+                [os.path.join(inst_dir, 'bin', 'clang-cl.exe'), '-m32'] + cc[1:],
+                [os.path.join(inst_dir, 'bin', 'clang-cl.exe'), '-m32'] + cxx[1:],
+                [os.path.join(inst_dir, 'bin', 'clang-cl.exe'), '-m32'] + asm[1:],
+                ld, ar, ranlib, libtool, compiler_rt_inst_dir)
+            cmake_args += [
+                '-DLLVM_CONFIG_PATH=%s' % slashify_path(
+                    os.path.join(inst_dir, 'bin', 'llvm-config')),
+                os.path.join(src_dir, 'projects', 'compiler-rt'),
+            ]
+            build_package(compiler_rt_build_dir, cmake_args)
+            os.environ['LIB'] = old_lib
         install_import_library(build_dir, inst_dir)
         install_asan_symbols(build_dir, inst_dir)
 
 
 # Return the absolute path of a build tool.  We first look to see if the
 # variable is defined in the config file, and if so we make sure it's an
 # absolute path to an existing tool, otherwise we look for a program in
 # $PATH named "key".
@@ -610,32 +660,34 @@ if __name__ == "__main__":
                 (cc_name, exe_ext)] + extra_cflags2,
             [stage1_inst_dir + "/bin/%s%s" %
                 (cxx_name, exe_ext)] + extra_cxxflags2,
             [stage1_inst_dir + "/bin/%s%s" %
                 (cc_name, exe_ext)] + extra_asmflags,
             [ld] + extra_ldflags,
             ar, ranlib, libtool,
             llvm_source_dir, stage2_dir, build_libcxx, osx_cross_compile,
-            build_type, assertions, python_path, gcc_dir, libcxx_include_dir)
+            build_type, assertions, python_path, gcc_dir, libcxx_include_dir,
+            stages == 2)
 
     if stages > 2:
         stage3_dir = build_dir + '/stage3'
         final_stage_dir = stage3_dir
         build_one_stage(
             [stage2_inst_dir + "/bin/%s%s" %
                 (cc_name, exe_ext)] + extra_cflags2,
             [stage2_inst_dir + "/bin/%s%s" %
                 (cxx_name, exe_ext)] + extra_cxxflags2,
             [stage2_inst_dir + "/bin/%s%s" %
                 (cc_name, exe_ext)] + extra_asmflags,
             [ld] + extra_ldflags,
             ar, ranlib, libtool,
             llvm_source_dir, stage3_dir, build_libcxx, osx_cross_compile,
-            build_type, assertions, python_path, gcc_dir, libcxx_include_dir)
+            build_type, assertions, python_path, gcc_dir, libcxx_include_dir,
+            stages == 3)
 
     package_name = "clang"
     if build_clang_tidy:
         prune_final_dir_for_clang_tidy(os.path.join(final_stage_dir, "clang"))
         package_name = "clang-tidy"
 
     if not args.skip_tar:
         ext = "bz2" if is_darwin() or is_windows() else "xz"
--- a/build/build-clang/clang-win64-st-an.json
+++ b/build/build-clang/clang-win64-st-an.json
@@ -11,12 +11,13 @@
     "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
     "python_path": "c:/mozilla-build/python/python.exe",
     "cc": "cl.exe",
     "cxx": "cl.exe",
     "ml": "ml64.exe",
     "patches": [
       "r318309.patch",
       "r320462.patch",
+      "r327876.patch",
       "loosen-msvc-detection.patch",
       "fflush-before-unlocking.patch"
     ]
 }
--- a/build/build-clang/clang-win64.json
+++ b/build/build-clang/clang-win64.json
@@ -10,11 +10,12 @@
     "compiler_repo": "https://llvm.org/svn/llvm-project/compiler-rt/trunk",
     "libcxx_repo": "https://llvm.org/svn/llvm-project/libcxx/trunk",
     "python_path": "c:/mozilla-build/python/python.exe",
     "cc": "cl.exe",
     "cxx": "cl.exe",
     "ml": "ml64.exe",
     "patches": [
       "loosen-msvc-detection.patch",
-      "r339636.patch"
+      "r339636.patch",
+      "workaround-issue38586.patch"
     ]
 }
new file mode 100644
--- /dev/null
+++ b/build/build-clang/r327876.patch
@@ -0,0 +1,39 @@
+From 522a892efc2ff22a2fd421b1ef4d9d9739d78b2e Mon Sep 17 00:00:00 2001
+From: Vitaly Buka <vitalybuka@google.com>
+Date: Mon, 19 Mar 2018 18:22:35 +0000
+Subject: [PATCH] Fix CMake/MSVC when compiler-rt and llvm are built separately
+
+Summary:
+For some reason CMake can't find the `append` macro if LLVM is built separately and imported via `LLVM_CONFIG_PATH`.
+
+Patch by Loo Rong Jie
+
+Reviewers: rnk, vitalybuka
+
+Reviewed By: rnk, vitalybuka
+
+Subscribers: dberris, mgorny, llvm-commits, #sanitizers
+
+Differential Revision: https://reviews.llvm.org/D43458
+
+git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@327876 91177308-0d34-0410-b5e6-96231b3b80d8
+---
+ CMakeLists.txt | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt
+index 4b953b212..a02ef9532 100644
+--- a/compiler-rt/CMakeLists.txt
++++ b/compiler-rt/CMakeLists.txt
+@@ -339,7 +339,7 @@ if (CMAKE_LINKER MATCHES "link.exe$")
+   # it, but CMake doesn't seem to have a way to set linker flags for
+   # individual static libraries, so we enable the suppression flag for
+   # the whole compiler-rt project.
+-  append("/IGNORE:4221" CMAKE_STATIC_LINKER_FLAGS)
++  set(CMAKE_STATIC_LINKER_FLAGS "${CMAKE_STATIC_LINKER_FLAGS} /IGNORE:4221")
+ endif()
+ 
+ add_subdirectory(include)
+-- 
+2.18.0
+
new file mode 100644
--- /dev/null
+++ b/build/build-clang/workaround-issue38586.patch
@@ -0,0 +1,31 @@
+diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt
+index 86ca2b3ef..f6ddd24eb 100644
+--- a/compiler-rt/CMakeLists.txt
++++ b/compiler-rt/CMakeLists.txt
+@@ -284,6 +284,26 @@ if(MSVC)
+   # warning from the MS linker complaining that it can't find the 'vc140.pdb'
+   # file used by our object library compilations.
+   list(APPEND SANITIZER_COMMON_CFLAGS /Z7)
++
++# Copied from llvm/cmake/modules/LLVMProcessSources.cmake
++function(llvm_replace_compiler_option var old new)
++  # Replaces a compiler option or switch `old' in `var' by `new'.
++  # If `old' is not in `var', appends `new' to `var'.
++  # Example: llvm_replace_compiler_option(CMAKE_CXX_FLAGS_RELEASE "-O3" "-O2")
++  # If the option already is on the variable, don't add it:
++  if( "${${var}}" MATCHES "(^| )${new}($| )" )
++    set(n "")
++  else()
++    set(n "${new}")
++  endif()
++  if( "${${var}}" MATCHES "(^| )${old}($| )" )
++    string( REGEX REPLACE "(^| )${old}($| )" " ${n} " ${var} "${${var}}" )
++  else()
++    set( ${var} "${${var}} ${n}" )
++  endif()
++  set( ${var} "${${var}}" PARENT_SCOPE )
++endfunction(llvm_replace_compiler_option)
++
+   llvm_replace_compiler_option(CMAKE_CXX_FLAGS "/Z[i7I]" "/Z7")
+   llvm_replace_compiler_option(CMAKE_CXX_FLAGS_DEBUG "/Z[i7I]" "/Z7")
+   llvm_replace_compiler_option(CMAKE_CXX_FLAGS_RELWITHDEBINFO "/Z[i7I]" "/Z7")