Bug 1391408 - Build minidump_stackwalk as a toolchain. r=froydnj
authorMike Hommey <mh+mozilla@glandium.org>
Fri, 11 Jan 2019 22:29:35 +0000
changeset 453577 a58fad772b6f940c06c65e8f8c686f5e39d09175
parent 453576 0e27ee5b44f8f576cb0f97fabaf8136cacc61786
child 453578 c45da646fc6140bd59a36bda18f6cecde6163feb
push id35360
push usernbeleuzu@mozilla.com
push dateSat, 12 Jan 2019 09:39:47 +0000
treeherdermozilla-central@cb35977ae7a4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfroydnj
bugs1391408
milestone66.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1391408 - Build minidump_stackwalk as a toolchain. r=froydnj This produces the same executables (produced for the same platforms) as those currently pulled from tooltool (modulo timestamps, maybe changes since last manifest change, etc.). Unfortunately, as of currently, the Windows variant needs to be cross-built with mingw because it doesn't compile without some POSIX APIs that MSVC/Windows SDK don't provide. One thing that is left out of this change is whether to be completely accurate with the toolchain cache hash (requiring a large list of files as resources, and making those built very frequently), whether we'd rely on manual updates, or if we should go with completely uncached tasks. This can be left for a followup, the tasks not being hooked up to be actually used by other tasks yet. Differential Revision: https://phabricator.services.mozilla.com/D16302
taskcluster/ci/toolchain/linux.yml
taskcluster/ci/toolchain/macosx.yml
taskcluster/ci/toolchain/windows.yml
taskcluster/docker/toolchain-build/Dockerfile
taskcluster/scripts/misc/build-minidump-stackwalk.sh
taskcluster/scripts/misc/minidump_stackwalk.sh
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@@ -50,17 +50,17 @@ linux64-clang-7:
         - linux64-gcc-4.9
 
 linux64-clang-trunk-mingw-x86:
     description: "MinGW-Clang Trunk x86 toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TMW(clang-x86)
-        tier: 2
+        tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux-xlarge
     worker:
         max-run-time: 7200
     run:
         using: toolchain-script
         script: build-clang-trunk-mingw.sh
         arguments: [
             'x86'
@@ -900,8 +900,30 @@ wrench-deps:
         script: wrench-deps-vendoring.sh
         sparse-profile: null  # need all of gfx/wr checked out for this script
         resources:
             - 'gfx/wr/Cargo.lock'
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/wrench-deps.tar.bz2
     toolchains:
         - linux64-rust-1.31  # whatever m-c is built with
+
+linux64-minidump-stackwalk:
+    description: "minidump_stackwalk toolchain build"
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TL(stackwalk)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        max-run-time: 1800
+    run:
+        using: toolchain-script
+        script: build-minidump-stackwalk.sh
+        sparse-profile: null
+        resources:
+            - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/build/minidump_stackwalk.tar.xz
+    toolchains:
+        - linux64-clang-7
+        - linux64-binutils
+        - linux64-rust-1.31
--- a/taskcluster/ci/toolchain/macosx.yml
+++ b/taskcluster/ci/toolchain/macosx.yml
@@ -178,8 +178,34 @@ macosx64-cbindgen:
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
         toolchain-artifact: public/build/cbindgen.tar.xz
     toolchains:
         - linux64-cctools-port
         - linux64-clang-7
         - linux64-llvm-dsymutil
         - linux64-rust-macos-1.31
+
+macosx64-minidump-stackwalk:
+    description: "minidump_stackwalk toolchain build"
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TM(stackwalk)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        max-run-time: 1800
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
+    run:
+        using: toolchain-script
+        script: build-minidump-stackwalk.sh
+        sparse-profile: null
+        arguments: ['macosx64']
+        tooltool-downloads: internal
+        resources:
+            - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/build/minidump_stackwalk.tar.xz
+    toolchains:
+        - linux64-cctools-port
+        - linux64-clang-7
+        - linux64-rust-macos-1.31
--- a/taskcluster/ci/toolchain/windows.yml
+++ b/taskcluster/ci/toolchain/windows.yml
@@ -331,8 +331,31 @@ win32-gn:
         script: build-gn-win32.sh
         resources:
             - 'taskcluster/scripts/misc/tooltool-download.sh'
             - 'taskcluster/scripts/misc/build-gn-common.sh'
         toolchain-artifact: public/build/gn.tar.bz2
     run-on-projects:
         - trunk
         - try
+
+win32-minidump-stackwalk:
+    description: "minidump_stackwalk toolchain build"
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TW32(stackwalk)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        docker-image: {in-tree: mingw32-build}
+        max-run-time: 1800
+    run:
+        using: toolchain-script
+        script: build-minidump-stackwalk.sh
+        sparse-profile: null
+        arguments: ['mingw32']
+        resources:
+            - 'taskcluster/scripts/misc/tooltool-download.sh'
+        toolchain-artifact: public/build/minidump_stackwalk.tar.xz
+    toolchains:
+        - linux64-clang-trunk-mingw-x86
+        - mingw32-rust-1.31
--- a/taskcluster/docker/toolchain-build/Dockerfile
+++ b/taskcluster/docker/toolchain-build/Dockerfile
@@ -10,26 +10,28 @@ ENV XZ_OPT=-T0
 
 # %ARG DOCKER_IMAGE_PACKAGES
 # %ARG TASKCLUSTER_ROOT_URL
 RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES
 
 RUN apt-get update && \
     apt-get install \
       autoconf \
+      autoconf2.13 \
       automake \
       bison \
       build-essential \
       curl \
       cmake \
       flex \
       gawk \
       gcc-multilib \
       gnupg \
       libbz2-dev \
+      libcurl4-openssl-dev \
       libssl-dev \
       libtool \
       ninja-build \
       p7zip-full \
       procps \
       python-pip \
       python-setuptools \
       python-virtualenv \
new file mode 100644
--- /dev/null
+++ b/taskcluster/scripts/misc/build-minidump-stackwalk.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+set -x -e -v
+
+# This script is for building minidump_stackwalk
+
+WORKSPACE=$HOME/workspace
+UPLOAD_DIR=$HOME/artifacts
+COMPRESS_EXT=xz
+
+cd $WORKSPACE/build/src
+
+. taskcluster/scripts/misc/tooltool-download.sh
+
+export MOZ_OBJDIR=obj-minidump
+
+echo ac_add_options --enable-project=tools/crashreporter > .mozconfig
+
+MINIDUMP_STACKWALK=minidump_stackwalk
+
+case "$1" in
+macosx64)
+    TOOLCHAINS="cctools rustc clang"
+    echo ac_add_options --target=x86_64-apple-darwin11 >> .mozconfig
+    echo ac_add_options --with-macos-sdk=$WORKSPACE/build/src/MacOSX10.11.sdk >> .mozconfig
+    ;;
+mingw32)
+    TOOLCHAINS="binutils rustc clang"
+    echo ac_add_options --target=i686-w64-mingw32 >> .mozconfig
+    echo export CC=i686-w64-mingw32-clang >> .mozconfig
+    echo export HOST_CC=clang >> .mozconfig
+    MINIDUMP_STACKWALK=minidump_stackwalk.exe
+    ;;
+*)
+    TOOLCHAINS="binutils rustc clang"
+    ;;
+esac
+
+for t in $TOOLCHAINS; do
+    PATH="$WORKSPACE/build/src/$t/bin:$PATH"
+done
+
+./mach build -v
+
+mkdir minidump_stackwalk
+cp $MOZ_OBJDIR/dist/bin/$MINIDUMP_STACKWALK minidump_stackwalk/
+
+tar -acf minidump_stackwalk.tar.$COMPRESS_EXT minidump_stackwalk/
+mkdir -p $UPLOAD_DIR
+cp minidump_stackwalk.tar.$COMPRESS_EXT $UPLOAD_DIR
deleted file mode 100755
--- a/taskcluster/scripts/misc/minidump_stackwalk.sh
+++ /dev/null
@@ -1,125 +0,0 @@
-#!/bin/bash
-#
-# This script builds minidump_stackwalk binaries from the Google Breakpad
-# source for all of the operating systems that we run Firefox tests on:
-# Linux x86, Linux x86-64, Windows x86, OS X x86-64.
-#
-# It expects to be run in the luser/breakpad-builder:0.7 Docker image and
-# needs access to the relengapiproxy to download internal tooltool files.
-
-set -v -e -x
-
-# This is a pain to support properly with gclient.
-#: BREAKPAD_REPO        ${BREAKPAD_REPO:=https://google-breakpad.googlecode.com/svn/trunk/}
-: BREAKPAD_REV         "${BREAKPAD_REV:=master}"
-: STACKWALK_HTTP_REPO  "${STACKWALK_HTTP_REPO:=https://hg.mozilla.org/users/tmielczarek_mozilla.com/stackwalk-http}"
-: STACKWALK_HTTP_REV   "${STACKWALK_HTTP_REV:=default}"
-
-ncpu=$(getconf _NPROCESSORS_ONLN)
-
-function build()
-{
-    cd /tmp
-    local platform=$1
-    local strip_prefix=$2
-    local configure_args=$3
-    local make_args=$4
-    local objdir=/tmp/obj-breakpad-$platform
-    local ext=
-    if test "$platform" = "win32"; then
-        ext=.exe
-    fi
-    rm -rf "$objdir"
-    mkdir "$objdir"
-    # First, build Breakpad
-    cd "$objdir"
-    # shellcheck disable=SC2086
-    CFLAGS="-O2 $CFLAGS" CXXFLAGS="-O2 $CXXFLAGS" /tmp/breakpad/src/configure --disable-tools $configure_args
-    # shellcheck disable=SC2086
-    make -j$ncpu $make_args src/libbreakpad.a src/third_party/libdisasm/libdisasm.a src/processor/stackwalk_common.o
-    # Second, build stackwalk-http
-    make -f /tmp/stackwalk-http/Makefile BREAKPAD_SRCDIR=/tmp/breakpad/src "BREAKPAD_OBJDIR=$(pwd)" "OS=$platform" "-j$ncpu"
-    "${strip_prefix}strip" "stackwalk${ext}"
-    cp "stackwalk${ext}" "/tmp/stackwalker/${platform}-minidump_stackwalk${ext}"
-}
-
-function linux64()
-{
-    export LDFLAGS="-static-libgcc -static-libstdc++"
-    build linux64
-    unset LDFLAGS
-}
-
-function linux32()
-{
-    export LDFLAGS="-static-libgcc -static-libstdc++ -L/tmp/libcurl-i386/lib"
-    export CFLAGS="-m32 -I/tmp/libcurl-i386/include"
-    export CXXFLAGS="-m32 -I/tmp/libcurl-i386/include"
-    build linux32 "" "--enable-m32"
-    unset LDFLAGS CFLAGS CXXFLAGS
-}
-
-function macosx64()
-{
-    cd /tmp
-    if ! test -d MacOSX10.7.sdk; then
-      python tooltool.py -v --manifest=macosx-sdk.manifest --url=http://relengapi/tooltool/ fetch
-    fi
-    export MACOSX_SDK=/tmp/MacOSX10.7.sdk
-    export CCTOOLS=/tmp/cctools
-    local FLAGS="-stdlib=libc++ -target x86_64-apple-darwin10 -mlinker-version=136 -B /tmp/cctools/bin -isysroot ${MACOSX_SDK} -mmacosx-version-min=10.7"
-    export CC="clang $FLAGS"
-    export CXX="clang++ $FLAGS -std=c++11"
-    local old_path="$PATH"
-    export PATH="/tmp/clang/bin:/tmp/cctools/bin/:$PATH"
-    export LD_LIBRARY_PATH=/usr/lib/llvm-3.6/lib/
-
-    build macosx64 "/tmp/cctools/bin/x86_64-apple-darwin10-" "--host=x86_64-apple-darwin10" "AR=/tmp/cctools/bin/x86_64-apple-darwin10-ar"
-
-    unset CC CXX LD_LIBRARY_PATH MACOSX_SDK CCTOOLS
-    export PATH="$old_path"
-}
-
-function win32()
-{
-    export LDFLAGS="-static-libgcc -static-libstdc++"
-    export CFLAGS="-D__USE_MINGW_ANSI_STDIO=1"
-    export CXXFLAGS="-D__USE_MINGW_ANSI_STDIO=1"
-    export ZLIB_DIR=/tmp/zlib-mingw
-    build win32 "i686-w64-mingw32-" "--host=i686-w64-mingw32"
-    unset LDFLAGS CFLAGS CXXFLAGS ZLIB_DIR
-}
-
-cd /tmp
-if ! test -d depot_tools; then
-  git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
-else
-  (cd depot_tools; git pull origin master)
-fi
-export PATH=$(pwd)/depot_tools:"$PATH"
-if ! test -d breakpad; then
-    mkdir breakpad
-    pushd breakpad
-    fetch breakpad
-    popd
-else
-    pushd breakpad/src
-    git pull origin master
-    popd
-fi
-pushd breakpad/src
-git checkout "${BREAKPAD_REV}"
-gclient sync
-popd
-
-(cd breakpad/src; git rev-parse master)
-if ! test -d stackwalk-http; then
-  hg clone -u "$STACKWALK_HTTP_REV" "$STACKWALK_HTTP_REPO"
-else
-  (cd stackwalk-http && hg pull "$STACKWALK_HTTP_REPO" && hg up "$STACKWALK_HTTP_REV")
-fi
-mkdir -p stackwalker
-linux64
-linux32
-macosx64
-win32