Bug 1623520 - Switch Windows asan and fuzzing builds to cross-builds. r=decoder
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 11 Dec 2020 05:06:12 +0000
changeset 560301 a1372a04b75abcfeda810a625d6dd97375720105
parent 560300 80b917c0690a2bca3806a740bb542c5ec5f75274
child 560302 7c01250bb4808442e19aef7bf6cfa3da4f3cc511
push id132568
push usermh@glandium.org
push dateFri, 11 Dec 2020 05:08:58 +0000
treeherderautoland@a1372a04b75a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdecoder
bugs1623520
milestone85.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1623520 - Switch Windows asan and fuzzing builds to cross-builds. r=decoder Differential Revision: https://phabricator.services.mozilla.com/D79417
build/win64/mozconfig.asan
taskcluster/ci/build/windows.yml
taskcluster/scripts/misc/repack-clang-linux-win-cross.sh
--- a/build/win64/mozconfig.asan
+++ b/build/win64/mozconfig.asan
@@ -1,14 +1,14 @@
 . "$topsrcdir/browser/config/mozconfigs/win64/common-win64"
 
 if [ -d "$MOZ_FETCHES_DIR/clang" ]; then
     CLANG_LIB_DIR="$(cd $MOZ_FETCHES_DIR/clang/lib/clang/* && cd lib/windows && pwd)"
 
-    export LIB=$LIB:$CLANG_LIB_DIR
+    export LIB=$CLANG_LIB_DIR
     export LDFLAGS="clang_rt.asan_dynamic-x86_64.lib clang_rt.asan_dynamic_runtime_thunk-x86_64.lib"
 
     export MOZ_COPY_PDBS=1
     export LLVM_SYMBOLIZER="$MOZ_FETCHES_DIR/clang/bin/llvm-symbolizer.exe"
     export MOZ_CLANG_RT_ASAN_LIB_PATH="${CLANG_LIB_DIR}/clang_rt.asan_dynamic-x86_64.dll"
 fi
 
 # Enable ASan specific code and build workarounds
--- a/taskcluster/ci/build/windows.yml
+++ b/taskcluster/ci/build/windows.yml
@@ -176,44 +176,57 @@ win64-fuzzing/debug:
         product: firefox
         job-name: win64-fuzzing-debug
     attributes:
         enable-full-crashsymbols: true
     treeherder:
         platform: windows2012-64/debug
         symbol: Bf
         tier: 1
-    worker-type: b-win2012
+    worker-type: b-linux
     worker:
+        docker-image: {in-tree: debian10-amd64-build}
         max-run-time: 7200
         env:
             PERFHERDER_EXTRA_OPTIONS: fuzzing
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
     run:
+        actions: [get-secrets, build]
         options: [append-env-variables-from-configs]
         script: mozharness/scripts/fx_desktop_build.py
         secrets: true
         config:
             - builds/releng_base_firefox.py
-            - builds/taskcluster_base_windows.py
-            - builds/taskcluster_base_win64.py
+            - builds/releng_base_linux_64_builds.py
             - builds/taskcluster_sub_win64/debug.py
+        extra-config:
+            mozconfig_platform: win64
+            env:
+                LD_PRELOAD: "/builds/worker/fetches/liblowercase/liblowercase.so"
+                LOWERCASE_DIRS: "/builds/worker/checkouts/gecko/vs2017_15.8.4"
         mozconfig-variant: debug-fuzzing
     run-on-projects: ['trunk']
     use-sccache: true
     fetches:
         toolchain:
-            - win64-clang-cl
-            - win64-rust
-            - win64-rust-size
-            - win64-cbindgen
-            - win64-sccache
-            - win64-nasm
-            - win64-node
-            - win64-winchecksec
+            - linux64-binutils
+            - linux64-clang-win-cross
+            - linux64-rust-cross
+            - linux64-rust-size
+            - linux64-nasm
+            - linux64-node
+            - linux64-cbindgen
+            - linux64-sccache
+            - linux64-dump-syms
+            - linux64-wine
+            - linux64-liblowercase
+            - linux64-winchecksec
+        fetch:
+            - nsis-3.01-win
+            - upx-3.95-win
     optimization:
         skip-unless-expanded: null
 
 win64-plain/debug:
     description: "Win64 Debug Plain"
     index:
         product: firefox
         job-name: win64-plain-debug
@@ -886,175 +899,220 @@ win64-asan/debug:
     description: "Win64 Debug ASAN"
     index:
         product: firefox
         job-name: win64-asan-debug
     treeherder:
         platform: windows2012-64/asan
         symbol: Bd
         tier: 1
-    worker-type: b-win2012
+    worker-type: b-linux
     worker:
+        docker-image: {in-tree: debian10-amd64-build}
         max-run-time: 7200
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
             PERFHERDER_EXTRA_OPTIONS: "debug asan"
     run:
+        actions: [get-secrets, build]
         options: [append-env-variables-from-configs]
         script: mozharness/scripts/fx_desktop_build.py
         secrets: true
         config:
             - builds/releng_base_firefox.py
-            - builds/taskcluster_base_windows.py
-            - builds/taskcluster_base_win64.py
+            - builds/releng_base_linux_64_builds.py
             - builds/taskcluster_sub_win64/asan_debug.py
+        extra-config:
+            mozconfig_platform: win64
+            env:
+                LD_PRELOAD: "/builds/worker/fetches/liblowercase/liblowercase.so"
+                LOWERCASE_DIRS: "/builds/worker/checkouts/gecko/vs2017_15.8.4"
         mozconfig-variant: debug-asan
     use-sccache: true
     fetches:
         toolchain:
-            - win64-clang-cl
-            - win64-rust
-            - win64-rust-size
-            - win64-cbindgen
-            - win64-sccache
-            - win64-nasm
-            - win64-node
-            # `fix-stacks` is needed because xpcshell self tests run as part of
-            # the build step. Once bug 1614626 moves them to the xpcshell task,
-            # this dependency can be removed.
-            - win32-fix-stacks
-            - win64-winchecksec
+            - linux64-binutils
+            - linux64-clang-win-cross
+            - linux64-rust-cross
+            - linux64-rust-size
+            - linux64-nasm
+            - linux64-node
+            - linux64-cbindgen
+            - linux64-sccache
+            - linux64-dump-syms
+            - linux64-wine
+            - linux64-liblowercase
+            - linux64-winchecksec
+        fetch:
+            - nsis-3.01-win
+            - upx-3.95-win
     optimization:
         skip-unless-expanded: null
 
 win64-asan/opt:
     description: "Win64 Opt ASAN"
     index:
         product: firefox
         job-name: win64-asan-opt
     treeherder:
         platform: windows2012-64/asan
         symbol: Bo
         tier: 1
-    worker-type: b-win2012
+    worker-type: b-linux
     worker:
+        docker-image: {in-tree: debian10-amd64-build}
         max-run-time: 7200
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
             PERFHERDER_EXTRA_OPTIONS: "opt asan"
             MOZ_AUTOMATION_PACKAGE_TESTS: "1"
     run:
+        actions: [get-secrets, build]
         options: [append-env-variables-from-configs]
         script: mozharness/scripts/fx_desktop_build.py
         secrets: true
         config:
             - builds/releng_base_firefox.py
-            - builds/taskcluster_base_windows.py
-            - builds/taskcluster_base_win64.py
+            - builds/releng_base_linux_64_builds.py
         extra-config:
             stage_platform: win64-asan
+            mozconfig_platform: win64
+            env:
+                LD_PRELOAD: "/builds/worker/fetches/liblowercase/liblowercase.so"
+                LOWERCASE_DIRS: "/builds/worker/checkouts/gecko/vs2017_15.8.4"
         mozconfig-variant: nightly-asan
     use-sccache: true
     fetches:
         toolchain:
-            - win64-clang-cl
-            - win64-rust
-            - win64-rust-size
-            - win64-cbindgen
-            - win64-sccache
-            - win64-nasm
-            - win64-node
-            - win64-winchecksec
+            - linux64-binutils
+            - linux64-clang-win-cross
+            - linux64-rust-cross
+            - linux64-rust-size
+            - linux64-nasm
+            - linux64-node
+            - linux64-cbindgen
+            - linux64-sccache
+            - linux64-dump-syms
+            - linux64-wine
+            - linux64-liblowercase
+            - linux64-winchecksec
+        fetch:
+            - nsis-3.01-win
+            - upx-3.95-win
 
 win64-asan-reporter-shippable/opt:
     description: "Win64 Opt ASAN Reporter (Shippable)"
     attributes:
         shippable: true
     shipping-product:
         by-release-type:
             nightly: firefox
             default: null
     index:
         product: firefox
         job-name: win64-asan-reporter-opt
         type: shippable
     treeherder:
         platform: win64-asan-reporter/opt
         symbol: BoR
-    worker-type: b-win2012
+    worker-type: b-linux
     worker:
+        docker-image: {in-tree: debian10-amd64-build}
         max-run-time: 7200
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
             PERFHERDER_EXTRA_OPTIONS: "asan-reporter"
             MOZ_AUTOMATION_PACKAGE_TESTS: "1"
     run:
+        actions: [get-secrets, build]
         options: [append-env-variables-from-configs]
         script: mozharness/scripts/fx_desktop_build.py
         secrets: true
         config:
             - builds/releng_base_firefox.py
-            - builds/taskcluster_base_windows.py
-            - builds/taskcluster_base_win64.py
+            - builds/releng_base_linux_64_builds.py
             - builds/taskcluster_sub_win64/asan_reporter_opt.py
             - taskcluster_nightly.py
         extra-config:
             stage_platform: win64-asan-reporter
+            mozconfig_platform: win64
+            env:
+                LD_PRELOAD: "/builds/worker/fetches/liblowercase/liblowercase.so"
+                LOWERCASE_DIRS: "/builds/worker/checkouts/gecko/vs2017_15.8.4"
         mozconfig-variant: nightly-asan-reporter
         mar-channel-id:
             firefox-mozilla-central-asan
         accepted-mar-channel-ids:
             firefox-mozilla-central-asan
     run-on-projects: ['mozilla-central']
     fetches:
         toolchain:
-            - win64-clang-cl
-            - win64-rust
-            - win64-rust-size
-            - win64-cbindgen
-            - win64-nasm
-            - win64-node
-            - win64-winchecksec
+            - linux64-binutils
+            - linux64-clang-win-cross
+            - linux64-rust-cross
+            - linux64-rust-size
+            - linux64-nasm
+            - linux64-node
+            - linux64-cbindgen
+            - linux64-dump-syms
+            - linux64-wine
+            - linux64-liblowercase
+            - linux64-winchecksec
+        fetch:
+            - nsis-3.01-win
+            - upx-3.95-win
 
 win64-asan-fuzzing/opt:
     description: "Win64 Fuzzing Opt ASAN"
     index:
         product: firefox
         job-name: win64-fuzzing-asan-opt
     treeherder:
         platform: windows2012-64/asan
         symbol: Bof
-    worker-type: b-win2012
+    worker-type: b-linux
     worker:
+        docker-image: {in-tree: debian10-amd64-build}
         max-run-time: 7200
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
             PERFHERDER_EXTRA_OPTIONS: fuzzing-asan
     run:
+        actions: [get-secrets, build]
         options: [append-env-variables-from-configs]
         script: mozharness/scripts/fx_desktop_build.py
         secrets: true
         config:
             - builds/releng_base_firefox.py
-            - builds/taskcluster_base_windows.py
-            - builds/taskcluster_base_win64.py
+            - builds/releng_base_linux_64_builds.py
         extra-config:
             stage_platform: win64-fuzzing-asan
+            mozconfig_platform: win64
+            env:
+                LD_PRELOAD: "/builds/worker/fetches/liblowercase/liblowercase.so"
+                LOWERCASE_DIRS: "/builds/worker/checkouts/gecko/vs2017_15.8.4"
         mozconfig-variant: nightly-fuzzing-asan
     use-sccache: true
     fetches:
         toolchain:
-            - win64-clang-cl
-            - win64-rust
-            - win64-rust-size
-            - win64-cbindgen
-            - win64-sccache
-            - win64-nasm
-            - win64-node
-            - win64-winchecksec
+            - linux64-binutils
+            - linux64-clang-win-cross
+            - linux64-rust-cross
+            - linux64-rust-size
+            - linux64-nasm
+            - linux64-node
+            - linux64-cbindgen
+            - linux64-sccache
+            - linux64-dump-syms
+            - linux64-wine
+            - linux64-liblowercase
+            - linux64-winchecksec
+        fetch:
+            - nsis-3.01-win
+            - upx-3.95-win
     optimization:
         skip-unless-expanded: null
 
 win32-devedition/opt:
     description: "Win32 Dev Edition (shippable)"
     use-pgo: win32-shippable/opt
     index:
         product: devedition
--- a/taskcluster/scripts/misc/repack-clang-linux-win-cross.sh
+++ b/taskcluster/scripts/misc/repack-clang-linux-win-cross.sh
@@ -4,15 +4,15 @@ set -x -e -v
 # This script is to repack a linux clang with Windows clang-cl.exe and compiler runtime.
 
 cd $MOZ_FETCHES_DIR
 
 # We already have the Linux clang extracted in $MOZ_FETCHES_DIR/clang by fetch-content
 # We have a non-extracted clang-cl/clang.tar.zst for Windows clang-cl that we need to extract
 # files from.
 
-$GECKO_PATH/taskcluster/scripts/misc/zstdpy -d clang-cl/clang.tar.zst | tar -x --wildcards clang/lib/clang/*/lib/windows clang/bin/clang-cl.exe
+$GECKO_PATH/taskcluster/scripts/misc/zstdpy -d clang-cl/clang.tar.zst | tar -x --wildcards clang/lib/clang/*/lib/windows clang/bin/clang-cl.exe clang/bin/llvm-symbolizer.exe
 chmod +x clang/bin/clang-cl.exe
 tar -c clang | $GECKO_PATH/taskcluster/scripts/misc/zstdpy > clang.tar.zst
 
 # Put a tarball in the artifacts dir
 mkdir -p $UPLOAD_DIR
 cp clang.tar.zst $UPLOAD_DIR