Improve Sisyphus error handling, bug 415911, not part of the build
authorbclary@bclary.com
Wed, 06 Feb 2008 12:00:53 -0800
changeset 11264 3b7306f399a83291e4c9d5712386009fd83181e1
parent 11263 a21623a24ba0cf49b5d8ee543ade87df83b51cc0
child 11265 afb8efc84e2f2e404acb99c4eb9304a59371e7b1
push id1
push userroot
push dateTue, 26 Apr 2011 22:38:44 +0000
treeherdermozilla-beta@bfdb6e623a36 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs415911
milestone1.9b4pre
Improve Sisyphus error handling, bug 415911, not part of the build
testing/sisyphus/bin/build.sh
testing/sisyphus/bin/builder.sh
testing/sisyphus/bin/check-spider.sh
testing/sisyphus/bin/checkout.sh
testing/sisyphus/bin/clean.sh
testing/sisyphus/bin/create-directory.sh
testing/sisyphus/bin/create-profile.sh
testing/sisyphus/bin/edit-talkback.sh
testing/sisyphus/bin/install-build.sh
testing/sisyphus/bin/install-extensions.sh
testing/sisyphus/bin/install-plugins.sh
testing/sisyphus/bin/library.sh
testing/sisyphus/bin/smoke-build.sh
testing/sisyphus/bin/test-setup.sh
testing/sisyphus/bin/tester.sh
testing/sisyphus/bin/uninstall-build.sh
--- a/testing/sisyphus/bin/build.sh
+++ b/testing/sisyphus/bin/build.sh
@@ -45,24 +45,24 @@ source $TEST_BIN/set-build-env.sh $@
 
 case $product in
     firefox|thunderbird)
         cd $TREE/mozilla
 
         if ! make -f client.mk build 2>&1; then
 
             if [[ -z "$TEST_FORCE_CLOBBER_ON_ERROR" ]]; then
-                error "error during build"
+                error "error during build" $LINENO
             else
-                echo "error occured during build. attempting a clobber build"
+                echo "error occured during build. attempting a clobber build" $LINENO
                 if ! make -f client.mk distclean 2>&1; then
-                    error "error during forced clobber"
+                    error "error during forced clobber" $LINENO
                 fi
                 if ! make -f client.mk build 2>&1; then
-                    error "error during forced build"
+                    error "error during forced build" $LINENO
                 fi
             fi
         fi
 
         case "$OSID" in
             mac) 
                 if [[ "$buildtype" == "debug" ]]; then
                     if [[ "$product" == "firefox" ]]; then
@@ -85,17 +85,17 @@ case $product in
 
         if [[ "$OSID" != "win32" ]]; then
             #
             # patch unix-like startup scripts to exec instead of 
             # forking new processes
             #
             executable=`get_executable $product $branch $executablepath`
             if [[ -z "$executable" ]]; then
-                error "get_executable $product $branch $executablepath returned empty path"
+                error "get_executable $product $branch $executablepath returned empty path" $LINENO
             fi
 
 
             executabledir=`dirname $executable`
 
             # patch to use exec to prevent forked processes
             cd "$executabledir"
             if [ -e "$product" ]; then
@@ -115,18 +115,18 @@ case $product in
 
     if [[ $buildtype == "debug" ]]; then
         export JSBUILDOPT=
     else
         export JSBUILDOPT=BUILD_OPT=1
     fi
 
     if ! make -f Makefile.ref ${JSBUILDOPT} clean 2>&1; then
-        error "during js/src clean"
+        error "during js/src clean" $LINENO
     fi 
 
     if ! make -f Makefile.ref ${JSBUILDOPT} 2>&1; then
-        error "during js/src build"
+        error "during js/src build" $LINENO
     fi
     ;;
 esac
 
 
--- a/testing/sisyphus/bin/builder.sh
+++ b/testing/sisyphus/bin/builder.sh
@@ -38,18 +38,16 @@
 # ***** END LICENSE BLOCK *****
 
 TEST_DIR=${TEST_DIR:-/work/mozilla/mozilla.com/test.mozilla.com/www}
 TEST_BIN=${TEST_BIN:-$TEST_DIR/bin}
 source ${TEST_BIN}/library.sh
 
 TEST_LOG=/dev/null
 
-trap 'echo -e "\n*** ERROR ***\n\b" && tail $TEST_LOG' ERR
-
 #
 # options processing
 #
 options="p:b:B:T:e:d:v"
 function usage()
 {
     cat<<EOF
 usage: 
--- a/testing/sisyphus/bin/check-spider.sh
+++ b/testing/sisyphus/bin/check-spider.sh
@@ -36,16 +36,18 @@
 # the terms of any one of the MPL, the GPL or the LGPL.
 #
 # ***** END LICENSE BLOCK *****
 
 TEST_DIR=${TEST_DIR:-/work/mozilla/mozilla.com/test.mozilla.com/www}
 TEST_BIN=${TEST_BIN:-$TEST_DIR/bin}
 source ${TEST_BIN}/library.sh
 
+TEST_STARTUP_TRIES=${TEST_STARTUP_TRIES:-3}
+
 #
 # options processing
 #
 options="p:b:x:N:d:"
 function usage()
 {
     cat <<EOF
 usage: 
@@ -63,17 +65,17 @@ variable            description
                     environment variable definitions to be included.
 
 note that the environment variables should have the same names as in the 
 "variable" column.
 
 Checks if the Spider extension is installed either in the named profile
 or as a global extension, by attempting up to 3 times to launch the Spider.
 
-If Spider fails to launch, the script returns exit code 66.
+If Spider fails to launch, the script returns exit code 2.
 
 EOF
     exit 1
 }
 
 unset product branch executablepath profilename datafiles
 
 while getopts $options optname ; 
@@ -96,45 +98,44 @@ if [[ -n "$datafiles" ]]; then
 fi
 
 if [[ -z "$product" || -z "$branch" || -z "$executablepath" || -z "$profilename" ]]; 
     then
     usage
 fi
 
 if [[ "$product" != "firefox" && "$product" != "thunderbird" ]]; then
-    error "product \"$product\" must be one of firefox or thunderbird"
+    error "product \"$product\" must be one of firefox or thunderbird" $LINENO
 fi
 
 if [[ "$branch" != "1.8.0" && "$branch" != "1.8.1" && "$branch" != "1.9.0" ]]; 
     then
-    error "branch \"$branch\" must be one of 1.8.0, 1.8.1, 1.9.0"
+    error "branch \"$branch\" must be one of 1.8.0, 1.8.1, 1.9.0" $LINENO
 fi
 
 executable=`get_executable $product $branch $executablepath`
 
 if [[ -z "$executable" ]]; then
-    error "get_executable $product $branch $executablepath returned empty path"
+    error "get_executable $product $branch $executablepath returned empty path" $LINENO
 fi
 
 if [[ ! -x "$executable" ]]; then 
-    error "executable \"$executable\" is not executable"
+    error "executable \"$executable\" is not executable" $LINENO
 fi
 
 if echo "$profilename" | egrep -qiv '[a-z0-9_]'; then
-    error "profile name must consist of letters, digits or _"
+    error "profile name must consist of letters, digits or _" $LINENO
 fi
 
 echo # attempt to force Spider to load
 
 tries=1
 while ! $TEST_BIN/timed_run.py ${TEST_STARTUP_TIMEOUT} "Start Spider: try $tries" \
     "$executable" -P "$profilename" \
     -spider -start -quit \
     -uri "http://${TEST_HTTP}/bin/start-spider.html" \
     -hook "http://${TEST_HTTP}/bin/userhook-checkspider.js"; do
   let tries=tries+1
-  if [ "$tries" -gt 3 ]; then
-      echo "Failed to start spider. Exiting..."
-      exit 66
+  if [ "$tries" -gt $TEST_STARTUP_TRIES  ]; then
+      error "Failed to start spider. Exiting..." $LINENO
   fi
 done
 
--- a/testing/sisyphus/bin/checkout.sh
+++ b/testing/sisyphus/bin/checkout.sh
@@ -39,53 +39,53 @@
 
 TEST_DIR=${TEST_DIR:-/work/mozilla/mozilla.com/test.mozilla.com/www}
 TEST_BIN=${TEST_BIN:-$TEST_DIR/bin}
 source ${TEST_BIN}/library.sh
 
 source ${TEST_BIN}/set-build-env.sh $@
 
 if [[ -z "$TREE" ]]; then
-    error "source tree not specified!"
+    error "source tree not specified!" $LINENO
 fi
 
 cd $TREE
 
 case $product in
     firefox|thunderbird)
         if [[ ! ( -d mozilla && \
             -e mozilla/client.mk && \
             -e "mozilla/$project/config/mozconfig" ) ]]; then
             if ! eval cvs -z3 -q co $BRANCH_CO_FLAGS \
                 mozilla/client.mk mozilla/$project/config/mozconfig; then
-                error "during checkout of mozconfig"
+                error "during checkout of mozconfig" $LINENO
             fi
         fi
 
         cd mozilla
 
         if ! make -f client.mk checkout 2>&1; then
-            error "during checkout of tree"
+            error "during checkout of tree" $LINENO
         fi
         ;;
 
     js) 
     if [[ ! ( -d mozilla && \
         -e mozilla/js && \
         -e mozilla/js/src ) ]]; then
         eval cvs -z3 -q co $BRANCH_CO_FLAGS $DATE_CO_FLAGS mozilla/js
     fi
 
     cd mozilla/js/src
 
     if ! eval cvs -z3 -q update $BRANCH_CO_FLAGS $DATE_CO_FLAGS -d -P 2>&1; then
-        error "during checkout of js/src"
+        error "during checkout of js/src" $LINENO
     fi
 
     if ! cvs -z3 -q update -d -P -A editline config  2>&1; then
-        error "during checkout of js/src"
+        error "during checkout of js/src" $LINENO
     fi
     # end for js shell
     ;;
     *)
-    error "unknown product $product"
+    error "unknown product $product" $LINENO
     ;;
 esac
--- a/testing/sisyphus/bin/clean.sh
+++ b/testing/sisyphus/bin/clean.sh
@@ -43,20 +43,20 @@ source ${TEST_BIN}/library.sh
 
 source ${TEST_BIN}/set-build-env.sh $@
 
 case $product in
     firefox|thunderbird)
         cd $TREE/mozilla
 
         if ! make -f client.mk clean 2>&1; then
-            error "during client.mk clean"
+            error "during client.mk clean" $LINENO
         fi
         ;;
 
     js)
         cd $TREE/mozilla/js/src
 
         if ! make -f Makefile.ref clean 2>&1; then
-            error "during SpiderMonkey clean"
+            error "during SpiderMonkey clean" $LINENO
         fi
         ;;
 esac
--- a/testing/sisyphus/bin/create-directory.sh
+++ b/testing/sisyphus/bin/create-directory.sh
@@ -86,34 +86,34 @@ while getopts $options optname ;
 done
 
 if [[ -z $directory ]]
     then
     usage
 fi
 
 if [[ `whoami` == "root" ]]; then
-    error "can not be run as root"
+    error "can not be run as root" $LINENO
 fi
 
 # get the cannonical name directory name
 mkdir -p "$directory"
 if ! pushd "$directory" > /dev/null ; then 
-    error "$directory is not accessible"
+    error "$directory is not accessible" $LINENO
 fi
 directory=`pwd`
 popd > /dev/null
 
 if [[ "$directory" == "/" ]]; then
-    error "directory $directory can not be root"
+    error "directory $directory can not be root" $LINENO
 fi
 
 parent=`dirname "$directory"`
 grandparent=`dirname "$parent"`
 
 if [[ "$parent" != "/tmp" && ( "$parent" == "/" || "$grandparent" == "/" ) ]]; then
-    error "directory $directory can not be a subdirectory of $parent"
+    error "directory $directory can not be a subdirectory of $parent" $LINENO
 fi
 
 
 # clean the directory if requested
 rm -fR $rmopt $directory
 mkdir -p "$directory"
--- a/testing/sisyphus/bin/create-profile.sh
+++ b/testing/sisyphus/bin/create-profile.sh
@@ -96,58 +96,58 @@ if [[ -n "$datafiles" ]]; then
 fi
 
 if [[ -z "$product" || -z "$branch" || -z "$executablepath" || \
     -z "$directory" || -z "$profilename" ]]; then
     usage
 fi
 
 if [[ "$product" != "firefox" && "$product" != "thunderbird" ]]; then
-    error "product \"$product\" must be one of firefox or thunderbird"
+    error "product \"$product\" must be one of firefox or thunderbird" $LINENO
 fi
 
 if [[ "$branch" != "1.8.0" && "$branch" != "1.8.1" && "$branch" != "1.9.0" ]]; 
     then
-    error "branch \"$branch\" must be one of 1.8.0, 1.8.1, 1.9.0"
+    error "branch \"$branch\" must be one of 1.8.0, 1.8.1, 1.9.0" $LINENO
 fi
 
 executable=`get_executable $product $branch $executablepath`
 
 if [[ -z "$executable" ]]; then
-    error "get_executable $product $branch $executablepath returned empty path"
+    error "get_executable $product $branch $executablepath returned empty path" $LINENO
 fi
 
 if [[ ! -x "$executable" ]]; then 
-    error "executable \"$executable\" is not executable"
+    error "executable \"$executable\" is not executable" $LINENO
 fi
 
 $TEST_BIN/create-directory.sh -d "$directory" -n 
 
 if echo "$profilename" | egrep -qiv '[a-z0-9_]'; then
-    error "profile name \"$profilename\" must consist of letters, digits or _"
+    error "profile name \"$profilename\" must consist of letters, digits or _" $LINENO
 fi
 
 if [ $OSID == "win32" ]; then
     directoryospath=`cygpath -a -w $directory`
     if [[ -z "$directoryospath" ]]; then
-	    error "unable to convert unix path to windows path"
+	    error "unable to convert unix path to windows path" $LINENO
     fi
 else
     directoryospath="$directory"
 fi
 
 echo "creating profile $profilename in directory $directory"
 
 if ! $TEST_BIN/timed_run.py ${TEST_STARTUP_TIMEOUT} "-" $executable -CreateProfile "$profilename $directoryospath"; then
-	error "creating profile $directory"
+	error "creating profile $directory" $LINENO
 fi
 
 if [[ -n $profiletemplate ]]; then
 	if [[ ! -d $profiletemplate ]]; then
-	    error "profile template directory $profiletemplate does not exist"
+	    error "profile template directory $profiletemplate does not exist" $LINENO
 	fi
 	echo "copying template profile $profiletemplate to $directory"
 	cp -R $profiletemplate/* $directory
 fi
 
 if [[ ! -z $user ]]; then
     cp $user $directory/user.js
 fi
--- a/testing/sisyphus/bin/edit-talkback.sh
+++ b/testing/sisyphus/bin/edit-talkback.sh
@@ -92,21 +92,21 @@ fi
 if [[ -z "$product" || -z "$branch" || \
     -z "$executablepath" || -z "$talkbackid" ]]; then
     usage
 fi
 
 executable=`get_executable $product $branch $executablepath`
 
 if [[ -z "$executable" ]]; then
-    error "get_executable $product $branch $executablepath returned empty path"
+    error "get_executable $product $branch $executablepath returned empty path" $LINENO
 fi
 
 if [[ ! -x "$executable" ]]; then 
-    error "executable \"$executable\" is not executable"
+    error "executable \"$executable\" is not executable" $LINENO
 fi
 
 executablepath=`dirname $executable`
 
 # escape & in talkback id to prevent replacements
 talkbackid=`echo $talkbackid | sed 's@&@\\\\&@g'`
 
 #
@@ -154,22 +154,22 @@ if [[ $talkback -eq 1 ]]; then
             vendorid=`grep '^VendorID = "' master.ini | sed 's@VendorID = "\([^"]*\)"@\1@'`
             productid=`grep '^ProductID = "' master.ini | sed 's@ProductID = "\([^"]*\)"@\1@'`
             platformid=`grep '^PlatformID = "' master.ini | sed 's@PlatformID = "\([^"]*\)"@\1@'`
             buildid=`grep '^BuildID = "' master.ini | sed 's@BuildID = "\([^"]*\)"@\1@'`
             talkbackdir="$HOME/Library/Application Support/FullCircle"
             IFS=:
             ;;
         *)
-            error "unknown os $OSID"
+            error "unknown os $OSID" $LINENO
             ;;
     esac
 
     if [[ -z "$talkbackdir" ]]; then
-        error "empty talkback directory"
+        error "empty talkback directory" $LINENO
     fi
 
     mkdir -p "$talkbackdir"
     
     case $OSID in
         win32)
             talkbackinidir="$talkbackdir/$vendorid/$productid/$platformid/$buildid"
             ;;
@@ -192,11 +192,11 @@ if [[ $talkback -eq 1 ]]; then
             ;;
         linux )
             sed -i.bak "s@URLEditControl .*@URLEditControl = \"mozqa:$talkbackid\"@" Talkback.ini
             ;;
         mac )
             sed -i.bak "s@URLEditControl .*@URLEditControl = \"mozqa:$talkbackid\"@" Talkback.ini
             ;;
         *)
-            error "unknown os=$OSID"
+            error "unknown os=$OSID" $LINENO
     esac
 fi
--- a/testing/sisyphus/bin/install-build.sh
+++ b/testing/sisyphus/bin/install-build.sh
@@ -106,61 +106,61 @@ if [[ $OSID == "win32" ]]; then
 	    if [[ $branch == "1.8.0" ]]; then
 	        $filename -ms -hideBanner -dd `cygpath -a -w "$executablepath"`
 	    else
 	        $filename /S /D=`cygpath -a -w "$executablepath"`
 	    fi
     elif echo  $filetype | grep -iq 'zip archive'; then
 	    unzip -o -d "$executablepath" "$filename"
     else
-	    error "$unknown file type $filetype"
+	    error "$unknown file type $filetype" $LINENO
     fi
 
 else
     
     case "$OSID" in
         linux)
         if echo $filetype | grep -iq 'bzip2'; then
             tar -jxvf $filename -C "$executablepath"
         elif echo $filetype | grep -iq 'gzip'; then
             tar -zxvf $filename -C "$executablepath" 
         else
-            error "unknown file type $filetype"
+            error "unknown file type $filetype" $LINENO
         fi
         ;; 
 
         mac)
         # answer license prompt
         result=`${TEST_BIN}/hdiutil-expect.ex $filename`
         # now get the volume data
 	    #result=`hdiutil attach $filename`
         disk=`echo $result | sed 's@.*\(/dev/[^ ]*\).*/dev.*/dev.*@\1@'`
         # remove the carriage return inserted by expect
         volume=`echo $result | sed "s|[^a-zA-Z0-9/]||g" | sed 's@.*\(/Volumes/.*\)@\1@'`
         echo "disk=$disk"
         echo "volume=$volume"
         if [[ -z "$disk" || -z "$volume" ]]; then
-            error "mounting disk image: $result"
+            error "mounting disk image: $result" $LINENO
         fi
 
         for app in $volume/*.app; do
             cp -R $app $executablepath
         done
 
         hdiutil detach $disk
         ;;
     esac
 
     #
     # patch unix-like startup scripts to exec instead of 
     # forking new processes
     #
     executable=`get_executable $product $branch $executablepath`
     if [[ -z "$executable" ]]; then
-        error "get_executable $product $branch $executablepath returned empty directory"
+        error "get_executable $product $branch $executablepath returned empty directory" $LINENO
     fi
     executabledir=`dirname $executable`
 
     # patch to use exec to prevent forked processes
     cd "$executabledir"
     if [ -e "$product" ]; then
 	    echo "$SCRIPT: patching $product"
 	    cp $TEST_BIN/$product.diff .
--- a/testing/sisyphus/bin/install-extensions.sh
+++ b/testing/sisyphus/bin/install-extensions.sh
@@ -95,40 +95,40 @@ if [[ -n "$datafiles" ]]; then
 fi
 
 if [[ -z "$product" || -z "$branch" || \
     -z "$executablepath" || -z "$profilename" || -z "$extensions" ]]; then
     usage
 fi
 
 if [[ "$product" != "firefox" && "$product" != "thunderbird" ]]; then
-    error "product \"$product\" must be one of firefox or thunderbird"
+    error "product \"$product\" must be one of firefox or thunderbird" $LINENO
 fi
 
 if [[ "$branch" != "1.8.0" && "$branch" != "1.8.1" && "$branch" != "1.9.0" ]]; 
     then
-    error "branch \"$branch\" must be one of 1.8.0, 1.8.1, 1.9.0"
+    error "branch \"$branch\" must be one of 1.8.0, 1.8.1, 1.9.0" $LINENO
 fi
 
 executable=`get_executable $product $branch $executablepath`
 
 if [[ -z "$executable" ]]; then
-    error "get_executable $product $branch $executablepath returned empty path"
+    error "get_executable $product $branch $executablepath returned empty path" $LINENO
 fi
 
 if [[ ! -x "$executable" ]]; then 
-    error "executable \"$executable\" is not executable"
+    error "executable \"$executable\" is not executable" $LINENO
 fi
 
 if echo $profilename | egrep -qiv '[a-z0-9_]'; then
-    error "profile name must consist of letters, digits or _"
+    error "profile name must consist of letters, digits or _" $LINENO
 fi
 
-for extension in $extensions/all/*; do 
-    if [[ $extension == "$extensions/all/*" ]]; then
+for extension in $extensions/all/*.xpi; do 
+    if [[ $extension == "$extensions/all/*.xpi" ]]; then
 	    break
     fi
     if [[ "$OSID" == "win32" ]]; then
         extensionos=`cygpath -a -w $extension`
     else
         extensionos="$extension"
     fi
 
@@ -145,25 +145,25 @@ for extension in $extensions/$OSID/*; do
     if [[ "$OSID" == "win32" ]]; then
         extensionos=`cygpath -a -w $extension`
     else
         extensionos="$extension"
     fi
 
     echo installing $extension
     if ! $TEST_BIN/timed_run.py ${TEST_STARTUP_TIMEOUT} "-" $executable -P $profilename -install-global-extension "$extensionos"; then
-        error "Failed to install $extensionos"
+        error "Failed to install $extensionos" $LINENO
     fi
 
 done
 
 # restart twice to make extension manager happy
 
 if ! $TEST_BIN/timed_run.py ${TEST_STARTUP_TIMEOUT} "install extensions - first restart" \
     $executable -P $profilename "http://${TEST_HTTP}/bin/install-extensions-1.html"; then
     echo "Ignoring 1st failure to load the install-extensions page"
 fi
 
 if ! $TEST_BIN/timed_run.py ${TEST_STARTUP_TIMEOUT} "install extensions - second restart" \
     $executable -P $profilename "http://${TEST_HTTP}/bin/install-extensions-2.html"; then
-    error "Fatal 2nd failure to load the install-extensions page"
+    error "Fatal 2nd failure to load the install-extensions page" $LINENO
 fi
 
--- a/testing/sisyphus/bin/install-plugins.sh
+++ b/testing/sisyphus/bin/install-plugins.sh
@@ -90,27 +90,27 @@ if [[ -n "$datafiles" ]]; then
 fi
 
 if [[ -z "$product" || -z "$branch" || \
     -z "$executablepath" || -z "$directory" ]]; then
     usage
 fi
 
 if [[ "$product" != "firefox" && "$product" != "thunderbird" ]]; then
-    error "product \"$product\" must be one of firefox or thunderbird"
+    error "product \"$product\" must be one of firefox or thunderbird" $LINENO
 fi
 
 executable=`get_executable $product $branch $executablepath`
 
 if [[ -z "$executable" ]]; then
-    error "get_executable $product $branch $executablepath returned empty path"
+    error "get_executable $product $branch $executablepath returned empty path" $LINENO
 fi
 
 if [[ ! -x "$executable" ]]; then 
-    error "executable \"$executable\" is not executable"
+    error "executable \"$executable\" is not executable" $LINENO
 fi
 
 executablepath=`dirname $executable`
 
 #
 # install plugins and components
 #
 echo "$SCRIPT: installing plugins from $directory/ in $executablepath/"
--- a/testing/sisyphus/bin/library.sh
+++ b/testing/sisyphus/bin/library.sh
@@ -38,33 +38,83 @@
 
 # This script contains a number of variables, functions, etc which 
 # are reused across a number of scripts. It should be included in each
 # script as follows:
 #
 # TEST_DIR=${TEST_DIR:-/work/mozilla/mozilla.com/test.mozilla.com/www}
 # TEST_BIN=${TEST_BIN:-$TEST_DIR/bin}
 # source ${TEST_BIN}/library.sh
-#
-#trap "echo error $0 `caller 1`; exit" ERR
-
-# skip remainder of script if it has already 
-# included
 
 if [[ -n "$DEBUG" ]]; then
     echo "calling $0 $@" 1>&2
 fi
 
+# export variables
+set -a 
+
+# in the event of an untrapped script error tail the test log, 
+# if it exists, to stderr then echo a FATAL ERROR message to the 
+# test log and stderr.
+
+function _err()
+{
+    local rc=$?
+    debug "_err: $0"
+
+    if [[ "$rc" -gt 0 ]]; then
+        if [[ -n "$TEST_LOG" ]]; then
+            echo -e "\nFATAL ERROR in $0 exit code $rc\n" >> $TEST_LOG
+        else
+            echo -e "\nFATAL ERROR in $0 exit code $rc\n" 1>&2
+        fi
+    fi
+    exit $rc
+}
+
+trap "_err" ERR
+
+function _exit()
+{
+    local rc=$?
+    local currscript=`get_scriptname $0`
+
+    debug "_exit: $0"
+
+    if [[ "$rc" -gt 0 && -n "$TEST_LOG" && "$SCRIPT" == "$currscript" ]]; then
+        # only tail the log once at the top level script
+        tail $TEST_LOG 1>&2
+    fi
+}
+
+trap "_exit" EXIT
+
+    # error message
+    # output error message end exit 2
+
+    error()
+    {
+        local message=$1
+        local lineno=$2
+
+        debug "error: $0:$LINENO"
+    
+        echo -e "FATAL ERROR in script $0:$lineno $message\n" 1>&2
+        if [[ "$0" == "-bash" || "$0" == "bash" ]]; then
+            return 0
+        fi
+        exit 2
+    } 
+
+
 if [[ -z "$LIBRARYSH" ]]; then
+    # skip remainder of script if it has already included
 
     LIBRARYSH=1
 
-    # export variables
-    set -a 
-
     # set time format for pipeline timing reports
     TIMEFORMAT="Elapsed time %0R seconds, User %0U seconds, System %0S seconds, CPU %P%%"
 
     MALLOC_CHECK_=2
 
     ulimit -c 0
 
     # debug msg
@@ -81,32 +131,20 @@ if [[ -z "$LIBRARYSH" ]]; then
     }
 
     # console msg
     #
     # output message to console, ie. stderr
 
     console()
     {
-        echo "$@" 1>&2
+        echo -e "$@" 1>&2
     }
 
 
-    # error message
-    # output error message end exit 2
-
-    error()
-    {
-        echo "error in script $SCRIPT: $1"
-        if [[ "$0" == "-bash" || "$0" == "bash" ]]; then
-            return 0
-        fi
-        exit 2
-    } 
-
     # dumpenvironment
     #
     # output environment to stdout
 
     dumpenvironment()
     {
         set | grep '^[A-Z]' | sed 's|^|environment: |'
     }
@@ -162,21 +200,28 @@ if [[ -z "$LIBRARYSH" ]]; then
             if find "$get_executable_directory" -perm +111 -type f \
                 -name "$get_executable_name" | \
                 grep "$get_executable_filter"; then
                 true
             fi
         fi
     }
 
-    if [[ "$0" == "-bash" || "$0" == "bash" ]]; then
-        SCRIPT="library.sh"
-    else
-        SCRIPT=`basename $0`
-    fi
+    function get_scriptname()
+    {
+        local script
+        if [[ "$0" == "-bash" || "$0" == "bash" ]]; then
+            script="library.sh"
+        else
+            script=`basename $0`
+        fi
+        echo $script
+    }
+
+    SCRIPT=`get_scriptname $0`
 
     TEST_DIR=${TEST_DIR:-/work/mozilla/mozilla.com/test.mozilla.com/www}
     TEST_BIN=${TEST_BIN:-$TEST_DIR/bin}
     TEST_HTTP=${TEST_HTTP:-test.mozilla.com}
     TEST_STARTUP_TIMEOUT=${TEST_STARTUP_TIMEOUT:-30}
 
     TEST_MACHINE=`uname -n`
     TEST_KERNEL=`uname -r`
--- a/testing/sisyphus/bin/smoke-build.sh
+++ b/testing/sisyphus/bin/smoke-build.sh
@@ -58,41 +58,41 @@ for filepath in $@; do
 	2) branch=1.8.1;;
 	3) branch=1.9.0;;
     esac
 
     echo $product $branch
 
     if ! install-build.sh  -p "$product" -b "$branch" -x "/tmp/$product-$branch" \
 	-f "$filepath"; then
-	error "installing build $product $branch into /tmp/$product-$branch"
+	error "installing build $product $branch into /tmp/$product-$branch" $LINENO
     fi
 
     if [[ "$product" == "thunderbird" ]]; then
 	template="-L ${TEST_DIR}/profiles/imap"
     else
 	unset template
     fi
 
     if ! create-profile.sh -p "$product" -b "$branch" \
 	-x "/tmp/$product-$branch" \
 	-D "/tmp/$product-$branch-profile" -N "$product-$branch-profile" \
 	-U ${TEST_DIR}/prefs/test-user.js \
 	$template; then
-	error "creating profile $product-$branch-profile at /tmp/$product-$branch"
+	error "creating profile $product-$branch-profile at /tmp/$product-$branch" $LINENO
     fi
 
     if ! install-extensions.sh -p "$product" -b "$branch" \
 	-x "/tmp/$product-$branch" \
 	-N "$product-$branch-profile" \
 	-E ${TEST_DIR}/xpi; then
-        error "installing extensions from ${TEST_DIR}/xpi"
+        error "installing extensions from ${TEST_DIR}/xpi" $LINENO
     fi
 
     if ! check-spider.sh -p "$product" -b "$branch" \
 	-x "/tmp/$product-$branch" \
 	-N "$product-$branch-profile"; then
-        error "check-spider.sh failed."
+        error "check-spider.sh failed." $LINENO
     fi
 
     uninstall-build.sh  -p "$product" -b "$branch" -x "/tmp/$product-$branch"
 
 done
--- a/testing/sisyphus/bin/test-setup.sh
+++ b/testing/sisyphus/bin/test-setup.sh
@@ -250,23 +250,15 @@ if [[ -n "$profiledirectory" ]]; then
 
     create-profile.sh -p $product -b $branch \
         -x $executablepath -D $profiledirectory -N $profilename \
         $optargs
 fi
 
 if [[ -n "$extensiondir" ]]; then
 
-    if ! install-extensions.sh -p $product -b $branch \
-        -x $executablepath -N $profilename -E $extensiondir; then
-        exit 2
-    fi
+    install-extensions.sh -p $product -b $branch \
+        -x $executablepath -N $profilename -E $extensiondir
 
-    if ! check-spider.sh -p $product -b $branch \
-        -x $executablepath -N $profilename; then
-        exit 2
-    fi
+    check-spider.sh -p $product -b $branch \
+        -x $executablepath -N $profilename
 
 fi
-
-
-
-
--- a/testing/sisyphus/bin/tester.sh
+++ b/testing/sisyphus/bin/tester.sh
@@ -1,9 +1,9 @@
-#!/bin/bash
+#!/bin/bash -e
 # -*- Mode: Shell-script; tab-width: 4; indent-tabs-mode: nil; -*-
 # ***** BEGIN LICENSE BLOCK *****
 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
 #
 # The contents of this file are subject to the Mozilla Public License Version
 # 1.1 (the "License"); you may not use this file except in compliance with
 # the License. You may obtain a copy of the License at
 # http://www.mozilla.org/MPL/
@@ -38,27 +38,25 @@
 # ***** END LICENSE BLOCK *****
 
 TEST_DIR=${TEST_DIR:-/work/mozilla/mozilla.com/test.mozilla.com/www}
 TEST_BIN=${TEST_BIN:-$TEST_DIR/bin}
 source ${TEST_BIN}/library.sh
 
 TEST_LOG=/dev/null
 
-#trap 'echo -e "\n*** ERROR ***\n\b" && tail $TEST_LOG' ERR
-
 #
 # options processing
 #
 options="p:b:e:T:t:v"
 function usage()
 {
     cat<<EOF
 usage: 
-$SCRIPT -t testscript datalist1 [datalist2 [datalist3 [datalist4]]]
+$SCRIPT -t testscript [-v ] datalist1 [datalist2 [datalist3 [datalist4]]]
 
 variable            description
 ===============     ===========================================================
 -t testscript       required. quoted test script with required arguments.
 -v                  optional. verbose - copies log file output to stdout.
 
 executes the testscript using the input data files in 
 $TEST_DIR/data constructed from each combination of the input parameters:
@@ -104,20 +102,16 @@ TEST_SUITE=`dirname $testscript | sed "s
 for data in $datalist; do
     TEST_DATE=`date -u +%Y-%m-%d-%H-%M-%S``date +%z`
     TEST_LOG="${TEST_DIR}/results/${TEST_DATE},$data,$OSID,${TEST_MACHINE},$TEST_SUITE.log"
 
     # tell caller what the log files are
     echo "log: $TEST_LOG "
 
     if [[ "$verbose" == "1" ]]; then
-        if ! test-setup.sh -d $TEST_DIR/data/$data.data 2>&1 | tee -a $TEST_LOG; then
-            error "test-setup.sh failed"
-        fi
+        test-setup.sh -d $TEST_DIR/data/$data.data 2>&1 | tee -a $TEST_LOG
         $testscript $testargs -d $TEST_DIR/data/$data.data 2>&1 | tee -a $TEST_LOG
     else
-        if ! test-setup.sh -d $TEST_DIR/data/$data.data >> $TEST_LOG 2>&1; then
-            error "test-setup.sh failed"
-        fi
+        test-setup.sh -d $TEST_DIR/data/$data.data >> $TEST_LOG 2>&1
         $testscript $testargs -d $TEST_DIR/data/$data.data >> $TEST_LOG 2>&1
     fi
 
 done
--- a/testing/sisyphus/bin/uninstall-build.sh
+++ b/testing/sisyphus/bin/uninstall-build.sh
@@ -137,17 +137,17 @@ if [[ $OSID == "win32" ]]; then
                 $uninstalloldexe /S /D=`cygpath -a -w $executabledir | sed 's@\\\\@\\\\\\\\@g'`
             else
                 uninstallexe="$executabledir/$product/uninstall/uninstaller.exe"
                 if [[ -n "$uninstallexe" && -e "$uninstallexe" ]]; then
                     $uninstallexe /S /D=`cygpath -a -w "$executabledir"  | sed 's@\\\\@\\\\\\\\@g'`
                 fi
             fi
         else
-            error "Unknown branch $branch"
+            error "Unknown branch $branch" $LINENO
         fi
         # the NSIS uninstaller will copy itself, then fork to the new
         # copy so that it can delete itself. This causes a race condition
         # between the uninstaller deleting the files and the rm command below
         # sleep for 10 seconds to give the uninstaller time to complete before
         # the installation directory is removed.
         sleep 10
     fi