Bug 1339989 - Create a sixgill compilation toolchain job, r=glandium
authorSteve Fink <sfink@mozilla.com>
Mon, 09 Oct 2017 17:49:29 -0700
changeset 386169 78eb19c1cbfab13082cfa36780322be30e7fc9b4
parent 386168 0c3f75e78a5b0aab8ad20fbab2a5767b99f1c9c9
child 386170 4f11834e48fc44b0ff1a7ce6446155a9a7ef0ed7
push id32676
push userarchaeopteryx@coole-files.de
push dateFri, 13 Oct 2017 21:38:18 +0000
treeherdermozilla-central@a31334a65a1c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1339989
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1339989 - Create a sixgill compilation toolchain job, r=glandium
browser/config/tooltool-manifests/linux64/hazard.manifest
taskcluster/ci/hazard/kind.yml
taskcluster/ci/toolchain/linux.yml
taskcluster/scripts/builder/hazard-analysis.sh
taskcluster/scripts/misc/build-gcc-sixgill-plugin-linux.sh
deleted file mode 100644
--- a/browser/config/tooltool-manifests/linux64/hazard.manifest
+++ /dev/null
@@ -1,26 +0,0 @@
-[
-  {
-    "digest": "2e56a3cf84764b8e63720e5f961cff7ba8ba5cf2f353dac55c69486489bcd89f53a757e09469a07700b80cd09f09666c2db4ce375b67060ac3be967714597231",
-    "unpack": true,
-    "algorithm": "sha512",
-    "filename": "sixgill.tar.xz",
-    "size": 2629600,
-    "hg_id": "221d0d2eead9"
-  },
-  {
-    "digest": "3915f8ec396c56a8a92e6f9695b70f09ce9d1582359d1258e37e3fd43a143bc974410e4cfc27f500e095f34a8956206e0ebf799b7287f0f38def0d5e34ed71c9",
-    "unpack": true,
-    "setup": "setup.sh",
-    "algorithm": "sha512",
-    "filename": "gtk3.tar.xz",
-    "size": 12072532
-  },
-  {
-    "version": "rustc 1.19.0 (0ade33941 2017-07-17) repack",
-    "size": 161014632,
-    "digest": "65bebcf94fc66ea618c58c9ac33f0f206095ecfe3931cc6edb301f4b40480e3b44b0f39aea7a25fed8eef47e63523e7e670082947a3662cdc04c68ebbe5dfc89",
-    "algorithm": "sha512",
-    "filename": "rustc.tar.xz",
-    "unpack": true
-  }
-]
--- a/taskcluster/ci/hazard/kind.yml
+++ b/taskcluster/ci/hazard/kind.yml
@@ -28,42 +28,46 @@ jobs:
         index:
             product: firefox
             job-name: shell-haz-debug
         treeherder:
             platform: linux64/debug
             symbol: SM-tc(H)
         worker:
             env:
-                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/hazard.manifest"
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
         run:
             using: hazard
             command: >
                 cd /builds/worker/checkouts/gecko/taskcluster/scripts/builder
                 && ./build-haz-linux.sh --project shell $HOME/workspace
         when:
             files-changed:
                 - js/public/**
                 - js/src/**
         toolchains:
             - linux64-clang
             - linux64-gcc-4.9
+            - linux64-sixgill
+            - linux64-rust
 
     linux64-haz/debug:
         description: "Browser Hazard Analysis Linux"
         index:
             product: firefox
             job-name: browser-haz-debug
         treeherder:
             platform: linux64/debug
             symbol: tc(H)
         worker:
             env:
-                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/hazard.manifest"
+                TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
         run:
             using: hazard
             mozconfig: "browser/config/mozconfigs/linux64/hazards"
             command: >
                 cd /builds/worker/checkouts/gecko/taskcluster/scripts/builder
                 && ./build-haz-linux.sh --project browser $HOME/workspace
         toolchains:
             - linux64-clang
             - linux64-gcc-4.9
+            - linux64-sixgill
+            - linux64-rust
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@@ -105,16 +105,37 @@ linux64-gcc-6:
     run:
         using: toolchain-script
         script: build-gcc-6-linux.sh
         resources:
             - 'build/unix/build-gcc/build-gcc.sh'
         toolchain-alias: linux64-gcc
         toolchain-artifact: public/build/gcc.tar.xz
 
+linux64-sixgill:
+    description: "sixgill GCC plugin build"
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TL(sixgill)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        docker-image: {in-tree: desktop-build}
+        max-run-time: 3600
+    run:
+        using: toolchain-script
+        script: build-gcc-sixgill-plugin-linux.sh
+        resources:
+            - 'build/unix/build-gcc/build-gcc.sh'
+            - 'taskcluster/scripts/misc/build-gcc-sixgill-plugin-linux.sh'
+        toolchain-artifact: public/build/sixgill.tar.xz
+    toolchains:
+        - linux64-gcc-4.9
+
 linux64-binutils:
     description: "Binutils toolchain build"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: TL(binutil)
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
--- a/taskcluster/scripts/builder/hazard-analysis.sh
+++ b/taskcluster/scripts/builder/hazard-analysis.sh
@@ -141,30 +141,43 @@ function grab_artifacts () {
 }
 
 function check_hazards () {
     (
     set +e
     NUM_HAZARDS=$(grep -c 'Function.*has unrooted.*live across GC call' "$1"/rootingHazards.txt)
     NUM_UNSAFE=$(grep -c '^Function.*takes unsafe address of unrooted' "$1"/refs.txt)
     NUM_UNNECESSARY=$(grep -c '^Function.* has unnecessary root' "$1"/unnecessary.txt)
+    NUM_DROPPED=$(grep -c '^Dropped CFG' "$1"/build_xgill.log)
     NUM_WRITE_HAZARDS=$(perl -lne 'print $1 if m!found (\d+)/\d+ allowed errors!' "$1"/heapWriteHazards.txt)
 
     set +x
     echo "TinderboxPrint: rooting hazards<br/>$NUM_HAZARDS"
     echo "TinderboxPrint: (unsafe references to unrooted GC pointers)<br/>$NUM_UNSAFE"
     echo "TinderboxPrint: (unnecessary roots)<br/>$NUM_UNNECESSARY"
     echo "TinderboxPrint: heap write hazards<br/>$NUM_WRITE_HAZARDS"
 
+    exit_status=0
+
     if [ $NUM_HAZARDS -gt 0 ]; then
         echo "TEST-UNEXPECTED-FAIL $NUM_HAZARDS rooting hazards detected" >&2
         echo "TinderboxPrint: documentation<br/><a href='https://wiki.mozilla.org/Javascript:Hazard_Builds#Diagnosing_a_rooting_hazards_failure'>static rooting hazard analysis failures</a>, visit \"Inspect Task\" link for hazard details"
-        exit 1
+        exit_status=1
     fi
 
     NUM_ALLOWED_WRITE_HAZARDS=0
     if [ $NUM_WRITE_HAZARDS -gt $NUM_ALLOWED_WRITE_HAZARDS ]; then
         echo "TEST-UNEXPECTED-FAIL $NUM_WRITE_HAZARDS heap write hazards detected out of $NUM_ALLOWED_WRITE_HAZARDS allowed" >&2
         echo "TinderboxPrint: documentation<br/><a href='https://wiki.mozilla.org/Javascript:Hazard_Builds#Diagnosing_a_heap_write_hazard_failure'>heap write hazard analysis failures</a>, visit \"Inspect Task\" link for hazard details"
-        exit 1
+        exit_status = 1
+    fi
+
+    if [ $NUM_DROPPED -gt 0 ]; then
+        echo "TEST-UNEXPECTED-FAIL $NUM_DROPPED CFGs dropped" >&2
+        echo "TinderboxPrint: sixgill unable to handle constructs<br/>$NUM_DROPPED"
+        exit_status=1
+    fi
+
+    if [ $exit_status -ne 0 ]; then
+        exit $exit_status
     fi
     )
 }
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/misc/build-gcc-sixgill-plugin-linux.sh
@@ -0,0 +1,105 @@
+#!/bin/bash
+
+set -e
+set -x
+
+# This script is for building the sixgill GCC plugin for Linux. It relies on
+# the gcc checkout because it needs to recompile gmp and the gcc build script
+# determines the version of gmp to download.
+
+WORKSPACE=$HOME/workspace
+HOME_DIR=$WORKSPACE/build
+UPLOAD_DIR=$HOME/artifacts
+
+root_dir=$HOME_DIR
+build_dir=$HOME_DIR/src/build
+data_dir=$HOME_DIR/src/build/unix/build-gcc
+
+# Download and unpack upstream toolchain artifacts (ie, the gcc binary).
+. $(dirname $0)/tooltool-download.sh
+
+gcc_version=4.9.4
+gcc_ext=bz2
+binutils_version=2.25.1
+binutils_ext=bz2
+sixgill_rev=59b74c2e21bd
+sixgill_repo=https://hg.mozilla.org/users/sfink_mozilla.com/sixgill
+
+. $data_dir/build-gcc.sh
+
+# GPG key used to sign GCC
+$GPG --import $data_dir/13975A70E63C361C73AE69EF6EEB81F8981C74C7.key
+# GPG key used to sign binutils
+$GPG --import $data_dir/EAF1C276A747E9ED86210CBAC3126D3B4AE55E93.key
+# GPG key used to sign GMP
+$GPG --import $data_dir/343C2FF0FBEE5EC2EDBEF399F3599FF828C67298.key
+# GPG key used to sign MPFR
+$GPG --import $data_dir/07F3DBBECC1A39605078094D980C197698C3739D.key
+# GPG key used to sign MPC
+$GPG --import $data_dir/AD17A21EF8AED8F1CC02DBD9F7D5C9BF765C61E3.key
+
+cat > $HOME_DIR/checksums <<EOF
+b5b14added7d78a8d1ca70b5cb75fef57ce2197264f4f5835326b0df22ac9f22  binutils-2.25.1.tar.bz2
+02500a4edd14875f94fe84cbeda4290425cb0c1c2474c6f75d75a303d64b4196  cloog-0.18.1.tar.gz
+6c11d292cd01b294f9f84c9a59c230d80e9e4a47e5c6355f046bb36d4f358092  gcc-4.9.4.tar.bz2
+752079520b4690531171d0f4532e40f08600215feefede70b24fabdc6f1ab160  gmp-5.1.3.tar.bz2
+f4b3dbee9712850006e44f0db2103441ab3d13b406f77996d1df19ee89d11fb4  isl-0.12.2.tar.bz2
+ae79f8d41d8a86456b68607e9ca398d00f8b7342d1d83bcf4428178ac45380c7  mpc-0.8.2.tar.gz
+ca498c1c7a74dd37a576f353312d1e68d490978de4395fa28f1cbd46a364e658  mpfr-3.1.5.tar.bz2
+EOF
+
+# Download GCC + related, and unpack.
+prepare
+
+export TMPDIR=${TMPDIR:-/tmp/}
+export gcc_bindir=$root_dir/src/gcc/bin
+export gmp_prefix=/tools/gmp
+export gmp_dir=$root_dir$gmp_prefix
+
+prepare_sixgill() {(
+    cd $root_dir
+    hg clone -r $sixgill_rev $sixgill_repo || ( cd sixgill && hg update -r $sixgill_rev )
+)}
+
+build_gmp() {
+    if ! [ -x $gcc_bindir/gcc ]; then
+        echo "GCC not found in $gcc_bindir/gcc" >&2
+        exit 1
+    fi
+
+    # The sixgill plugin uses some gmp symbols, including some not exported by
+    # cc1/cc1plus. So link the plugin statically to libgmp. Except that the
+    # default static build does not have -fPIC, and will result in a relocation
+    # error, so build our own. This requires the gcc and related source to be
+    # in $root_dir/gcc-$gcc_version.
+
+    mkdir $root_dir/gmp-objdir || true
+    (
+        cd $root_dir/gmp-objdir
+        $root_dir/gcc-$gcc_version/gmp/configure --disable-shared --with-pic --prefix=$gmp_prefix
+        make -j8
+        make install DESTDIR=$root_dir
+    )
+}
+
+build_sixgill() {(
+    cd $root_dir/sixgill
+    export CC=$gcc_bindir/gcc
+    export CXX=$gcc_bindir/g++
+    export TARGET_CC=$CC
+    export CPPFLAGS=-I$gmp_dir/include
+    export EXTRA_LDFLAGS=-L$gmp_dir/lib
+    export HOST_CFLAGS=$CPPFLAGS
+
+    ./release.sh --build-and-package --with-gmp=$gmp_dir
+    tarball=$(ls -td *-sixgill | head -1)/sixgill.tar.xz
+    cp $tarball $root_dir/sixgill.tar.xz
+)}
+
+prepare_sixgill
+build_gmp
+build_sixgill
+
+# Put a tarball in the artifacts dir
+mkdir -p $UPLOAD_DIR
+cp $HOME_DIR/sixgill.tar.* $UPLOAD_DIR