Bug 1250709 - Taskcluster-based browser rooting hazard analysis, r=garndt
☠☠ backed out by 7279620cb6bb ☠ ☠
authorSteve Fink <sfink@mozilla.com>
Mon, 15 Feb 2016 21:06:42 -0800
changeset 288699 b5bffc3757191411f15de9f8ed432f64fe69703c
parent 288698 c73617bdfdfa52f98c33b46586eec0ad2e695193
child 288700 66445d3d11dea815db8888552910e7f836749d27
push id18174
push usercbook@mozilla.com
push dateTue, 15 Mar 2016 09:44:58 +0000
treeherderfx-team@dd0baa33759d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgarndt
bugs1250709
milestone48.0a1
Bug 1250709 - Taskcluster-based browser rooting hazard analysis, r=garndt
browser/config/mozconfigs/linux64/hazards
testing/taskcluster/scripts/builder/build-browser-haz-linux.sh
testing/taskcluster/scripts/builder/get-objdir.py
testing/taskcluster/scripts/builder/hazard-analysis.sh
testing/taskcluster/tasks/branches/try/job_flags.yml
testing/taskcluster/tasks/builds/haz_linux.yml
--- a/browser/config/mozconfigs/linux64/hazards
+++ b/browser/config/mozconfigs/linux64/hazards
@@ -1,10 +1,10 @@
-# This mozconfig is used when compiling the browser for the SM(Hf) rooting
-# hazard analysis build, see
+# This mozconfig is used when compiling the browser for the rooting hazard
+# analysis build (labeled H on treeherder). See
 # https://wiki.mozilla.org/Javascript:SpiderMonkey:ExactStackRooting
 
 # Do NOT include build/unix/mozconfig.linux because it points directly at the
 # tooltool-installed gcc, and the analysis works by wrapping the gcc invocation
 # with a script that invokes the real gcc with -fplugin and its configuration
 # directives. Instead, duplicate the contents of that mozconfig here:
 
 . "$topsrcdir/build/mozconfig.common"
@@ -16,18 +16,19 @@ ac_add_options --enable-stdcxx-compat
 mk_add_options MOZ_OBJDIR=obj-analyzed
 
 # The configuration options are chosen to compile the most code
 # (--enable-debug, --enable-tests) in the trickiest way possible
 # (--enable-optimize) to maximize the chance of seeing tricky static orderings.
 ac_add_options --enable-debug
 ac_add_options --enable-tests
 ac_add_options --enable-optimize
+ac_add_options --with-compiler-wrapper=$(cd $topsrcdir/sixgill/usr/libexec/sixgill/scripts/wrap_gcc && echo $PWD)/basecc
+ac_add_options --without-ccache
 
 CFLAGS="$CFLAGS -Wno-attributes"
 CPPFLAGS="$CPPFLAGS -Wno-attributes"
 CXXFLAGS="$CXXFLAGS -Wno-attributes"
 
-TOOLTOOL_DIR="$(dirname $topsrcdir)"
 export PKG_CONFIG_LIBDIR=/usr/lib64/pkgconfig:/usr/share/pkgconfig
 . $topsrcdir/build/unix/mozconfig.gtk
 
 . "$topsrcdir/build/mozconfig.common.override"
new file mode 100755
--- /dev/null
+++ b/testing/taskcluster/scripts/builder/build-browser-haz-linux.sh
@@ -0,0 +1,27 @@
+#!/bin/bash -ex
+
+################################### build-mulet-haz-linux.sh ###################################
+# Ensure all the scripts in this dir are on the path....
+DIRNAME=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+PATH=$DIRNAME:$PATH
+
+. desktop-setup.sh
+. hazard-analysis.sh
+
+build_js_shell
+
+# Artifacts folder is outside of the cache.
+mkdir -p $HOME/artifacts/ || true
+
+function onexit () {
+    grab_artifacts "$WORKSPACE/analysis" "$HOME/artifacts"
+}
+
+trap onexit EXIT
+
+configure_analysis "$WORKSPACE/analysis"
+run_analysis "$WORKSPACE/analysis" browser
+
+check_hazards "$WORKSPACE/analysis"
+
+################################### script end ###################################
--- a/testing/taskcluster/scripts/builder/get-objdir.py
+++ b/testing/taskcluster/scripts/builder/get-objdir.py
@@ -1,9 +1,9 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2.7
 
 from __future__ import print_function
 import sys
 import os
 import json
 import subprocess
 from StringIO import StringIO
 
--- a/testing/taskcluster/scripts/builder/hazard-analysis.sh
+++ b/testing/taskcluster/scripts/builder/hazard-analysis.sh
@@ -1,18 +1,27 @@
 #!/bin/bash -ex
 
-HAZARD_SHELL_OBJDIR=$MOZ_OBJDIR/obj-haz-shell
+[ -n "$WORKSPACE" ]
+[ -n "$MOZ_OBJDIR" ]
+[ -n "$GECKO_DIR" ]
+
+HAZARD_SHELL_OBJDIR=$WORKSPACE/obj-haz-shell
 JS_SRCDIR=$GECKO_DIR/js/src
 ANALYSIS_SRCDIR=$JS_SRCDIR/devtools/rootAnalysis
 
 # Install the sixgill tool
 TOOLTOOL_MANIFEST=js/src/devtools/rootAnalysis/build/sixgill.manifest
 . install-packages.sh "$GECKO_DIR"
 
+PYTHON=python2.7
+if ! which $PYTHON; then
+    PYTHON=python
+fi
+
 export CC="$GECKO_DIR/gcc/bin/gcc"
 export CXX="$GECKO_DIR/gcc/bin/g++"
 
 function build_js_shell () {
     ( cd $JS_SRCDIR; autoconf-2.13 )
     mkdir -p $HAZARD_SHELL_OBJDIR || true
     cd $HAZARD_SHELL_OBJDIR
     $JS_SRCDIR/configure --enable-optimize --disable-debug --enable-ctypes --enable-nspr-build --without-intl-api --with-ccache
@@ -40,42 +49,56 @@ EOF
 function run_analysis () {
     local analysis_dir
     analysis_dir="$1"
     local build_type
     build_type="$2"
 
     (
         cd "$analysis_dir"
-        python "$ANALYSIS_SRCDIR/analyze.py" --buildcommand="$GECKO_DIR/testing/mozharness/scripts/spidermonkey/build.${build_type}"
+        $PYTHON "$ANALYSIS_SRCDIR/analyze.py" --buildcommand="$GECKO_DIR/testing/mozharness/scripts/spidermonkey/build.${build_type}"
     )
 }
 
 function grab_artifacts () {
     local analysis_dir
     analysis_dir="$1"
     local artifacts
     artifacts="$2"
 
     (
         cd "$analysis_dir"
         ls -lah
 
+        # Do not error out if no files found
+        shopt -s nullglob
+        set +e
         for f in *.txt *.lst; do
             gzip -9 -c "$f" > "${artifacts}/$f.gz"
         done
 
         # Check whether the user requested .xdb file upload in the top commit comment
 
-        if hg --cwd "$GECKO_DIR" log -l1 --template '{desc}\n' | grep -q 'haz: --upload-xdbs'; then
+        if hg --cwd "$GECKO_DIR" log -l1 --template '{desc}\n' | grep -q -- '--upload-xdbs'; then
             for f in *.xdb; do
                 bzip2 -c "$f" > "${artifacts}/$f.bz2"
             done
         fi
     )
 }
 
 function check_hazards () {
-    if grep 'Function.*has unrooted.*live across GC call' "$1"/rootingHazards.txt; then
-        echo "TEST-UNEXPECTED-FAIL hazards detected" >&2
+    (
+    set +e
+    NUM_HAZARDS=$(grep -c 'Function.*has unrooted.*live across GC call' "$1"/rootingHazards.txt)
+    NUM_UNSAFE=$(grep -c '^Function.*takes unsafe address of unrooted' "$1"/refs.txt)
+    NUM_UNNECESSARY=$(grep -c '^Function.* has unnecessary root' "$1"/unnecessary.txt)
+
+    echo "TinderboxPrint: $NUM_HAZARDS rooting hazards"
+    echo "TinderboxPrint: $NUM_UNSAFE unsafe references to unrooted GC pointers"
+    echo "TinderboxPrint: $NUM_UNSAFE unnecessary roots"
+
+    if [ $NUM_HAZARDS -gt 0 ]; then
+        echo "TEST-UNEXPECTED-FAIL $NUM_HAZARDS hazards detected" >&2
         exit 1
     fi
+    )
 }
--- a/testing/taskcluster/tasks/branches/try/job_flags.yml
+++ b/testing/taskcluster/tasks/branches/try/job_flags.yml
@@ -70,16 +70,22 @@ builds:
       debug:
         task: tasks/builds/dbg_linux64_clobber.yml
   linux64-st-an:
     platforms:
       - Linux64 Static Analysis
     types:
       opt:
         task: tasks/builds/opt_linux64_st-an_clobber.yml
+  linux64-haz:
+    platforms:
+      - Linux64
+    types:
+      debug:
+        task: tasks/builds/haz_linux.yml
   macosx64:
     platforms:
       - MacOSX64
     types:
       opt:
         task: tasks/builds/opt_macosx64_clobber.yml
       debug:
         task: tasks/builds/dbg_macosx64_clobber.yml
new file mode 100644
--- /dev/null
+++ b/testing/taskcluster/tasks/builds/haz_linux.yml
@@ -0,0 +1,59 @@
+$inherits:
+  from: 'tasks/builds/firefox_base.yml'
+  variables:
+    build_name: 'browser-haz'
+    build_type: 'debug'
+task:
+  metadata:
+    name: '[TC] Browser Hazard Linux'
+    description: Browser Hazard Analysis Linux
+
+  workerType: dbg-linux64
+
+  scopes:
+    - 'docker-worker:cache:tooltool-cache'
+    - 'docker-worker:relengapi-proxy:tooltool.download.public'
+    - 'docker-worker:cache:level-{{level}}-{{project}}-build-linux64-haz-workspace'
+
+  payload:
+    cache:
+      level-{{level}}-{{project}}-build-linux64-haz-workspace: '/home/worker/workspace'
+      tooltool-cache: '/home/worker/tools/tooltool-cache'
+
+    features:
+      relengAPIProxy: true
+
+    env:
+      MOZCONFIG: 'browser/config/mozconfigs/linux64/hazards'
+      TOOLTOOL_CACHE: '/home/worker/tools/tooltool-cache'
+      TOOLTOOL_MANIFEST: 'browser/config/tooltool-manifests/linux64/releng.manifest'
+
+    maxRunTime: 36000
+
+    command:
+      - /bin/bash
+      - -c
+      - >
+        tc-vcs checkout workspace/gecko "$GECKO_BASE_REPOSITORY" "$GECKO_HEAD_REPOSITORY" "$GECKO_HEAD_REV" "$GECKO_HEAD_REF" &&
+        cd ./workspace/gecko/testing/taskcluster/scripts/builder &&
+        ./build-browser-haz-linux.sh $HOME/workspace
+
+  extra:
+    treeherderEnv:
+      - production
+      - staging
+    treeherder:
+      machine:
+        # see https://github.com/mozilla/treeherder/blob/master/ui/js/values.js
+        platform: linux64
+      groupSymbol: tc
+      groupName: Submitted by taskcluster for your pleasure
+      symbol: H
+      collection:
+        debug: true
+    # Rather then enforcing particular conventions we require that all build
+    # tasks provide the "build" extra field to specify where the build and tests
+    # files are located.
+    locations:
+      build: null
+      tests: null