Bug 1551743 Generate better treeherder errors when update-verify fails; r=bhearsum
authorTom Prince <mozilla@hocat.ca>
Wed, 15 May 2019 14:13:34 +0000
changeset 532782 c58e444ffd0cf281ed4a708c149c834fe66fbba3
parent 532781 0d390c4c20d6eb41bddf989346de8cf2470c85c9
child 532783 9b3e897d5a6ad159a1ab1e895b502fe0b2e40cbf
push id11272
push userapavel@mozilla.com
push dateThu, 16 May 2019 15:28:22 +0000
treeherdermozilla-beta@2265bfc5920d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbhearsum
bugs1551743
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1551743 Generate better treeherder errors when update-verify fails; r=bhearsum Treeherder will display a summary of errors from the log, if they are match certain patterns. Make update-verify more useful by outputing errors that match. Differential Revision: https://phabricator.services.mozilla.com/D31165
tools/update-verify/release/common/check_updates.sh
tools/update-verify/release/common/download_builds.sh
tools/update-verify/release/common/download_mars.sh
tools/update-verify/release/updates/verify.sh
tools/update-verify/scripts/chunked-verify.sh
--- a/tools/update-verify/release/common/check_updates.sh
+++ b/tools/update-verify/release/common/check_updates.sh
@@ -59,26 +59,26 @@ check_updates () {
 
     cd_dir=$(ls -d ${PWD}/source/${platform_dirname})
     cd "${cd_dir}"
     set -x
     "$updater" "$update_abspath" "$cwd" "$cwd" 0
     set +x
     cd ../..
   else
-    echo "FAIL: no dir in source/$platform_dirname"
+    echo "TEST-UNEXPECTED-FAIL: no dir in source/$platform_dirname"
     return 1
   fi
 
   cat update/update.log
   update_status=`cat update/update.status`
 
   if [ "$update_status" != "succeeded" ]
   then
-    echo "FAIL: update status was not successful: $update_status"
+    echo "TEST-UNEXPECTED-FAIL: update status was not successful: $update_status"
     return 1
   fi
 
   # If we were testing an OS X mar on Linux, the unpack step copied the
   # precomplete file from Contents/Resources to the root of the install
   # to ensure the Linux updater binary could find it. However, only the
   # precomplete file in Contents/Resources was updated, which means
   # the copied version in the root of the install will usually have some
@@ -99,16 +99,16 @@ check_updates () {
   fi
   cd ../..
 
   ../compare-directories.py source/${platform_dirname} target/${platform_dirname}  ${channel} > "${diff_file}"
   diffErr=$?
   cat "${diff_file}"
   if [ $diffErr == 2 ]
   then
-    echo "FAIL: differences found after update"
+    echo "TEST-UNEXPECTED-FAIL: differences found after update"
     return 1
   elif [ $diffErr != 0 ]
   then
-    echo "FAIL: unknown error from diff: $diffErr"
+    echo "TEST-UNEXPECTED-FAIL: unknown error from diff: $diffErr"
     return 3
   fi
 }
--- a/tools/update-verify/release/common/download_builds.sh
+++ b/tools/update-verify/release/common/download_builds.sh
@@ -21,16 +21,16 @@ download_builds() {
     do
     source_file=`basename "$url"`
     if [ -f "$source_file" ]; then rm "$source_file"; fi
     cd downloads 
     if [ -f "$source_file" ]; then rm "$source_file"; fi
     cached_download "${source_file}" "${url}"
     status=$?
     if [ $status != 0 ]; then
-      echo "FAIL: Could not download source $source_file from $url"
+      echo "TEST-UNEXPECTED-FAIL: Could not download source $source_file from $url"
       echo "skipping.."
       cd ../
       return $status
     fi
     cd ../
   done
 }
--- a/tools/update-verify/release/common/download_mars.sh
+++ b/tools/update-verify/release/common/download_mars.sh
@@ -26,48 +26,48 @@ download_mars () {
         try=$(($try+1))
     done
 
     echo; echo;  # padding
 
     update_line=`fgrep "<update " update.xml`
     grep_rv=$?
     if [ 0 -ne $grep_rv ]; then
-        echo "FAIL: no <update/> found for $update_url"
+        echo "TEST-UNEXPECTED-FAIL: no <update/> found for $update_url"
         return 1
     fi
     command=`echo $update_line | sed -e 's/^.*<update //' -e 's:>.*$::' -e 's:\&amp;:\&:g'`
     eval "export $command"
 
     if [ ! -z "$to_build_id" -a "$buildID" != "$to_build_id" ]; then
-        echo "FAIL: expected buildID $to_build_id does not match actual $buildID"
+        echo "TEST-UNEXPECTED-FAIL: expected buildID $to_build_id does not match actual $buildID"
         return 1
     fi
 
     if [ ! -z "$to_display_version" -a "$displayVersion" != "$to_display_version" ]; then
-        echo "FAIL: expected displayVersion $to_display_version does not match actual $displayVersion"
+        echo "TEST-UNEXPECTED-FAIL: expected displayVersion $to_display_version does not match actual $displayVersion"
         return 1
     fi
 
     if [ ! -z "$to_app_version" -a "$appVersion" != "$to_app_version" ]; then
-        echo "FAIL: expected appVersion $to_app_version does not match actual $appVersion"
+        echo "TEST-UNEXPECTED-FAIL: expected appVersion $to_app_version does not match actual $appVersion"
         return 1
     fi
 
     mkdir -p update/
     if [ -z $only ]; then
       only="partial complete"
     fi
     for patch_type in $only
       do
       line=`fgrep "patch type=\"$patch_type" update.xml`
       grep_rv=$?
 
       if [ 0 -ne $grep_rv ]; then
-        echo "FAIL: no $patch_type update found for $update_url"
+        echo "TEST-UNEXPECTED-FAIL: no $patch_type update found for $update_url"
         return 1
       fi
 
       command=`echo $line | sed -e 's/^.*<patch //' -e 's:/>.*$::' -e 's:\&amp;:\&:g'`
       eval "export $command"
 
       if [ "$test_only" == "1" ]
       then
@@ -80,26 +80,26 @@ download_mars () {
       if [ "$?" != 0 ]; then
         echo "Could not download $patch_type!"
         echo "from: $URL"
       fi
       actual_size=`perl -e "printf \"%d\n\", (stat(\"update/$patch_type.mar\"))[7]"`
       actual_hash=`openssl dgst -$hashFunction update/$patch_type.mar | sed -e 's/^.*= //'`
 
       if [ $actual_size != $size ]; then
-          echo "FAIL: $patch_type from $update_url wrong size"
-          echo "FAIL: update.xml size: $size"
-          echo "FAIL: actual size: $actual_size"
+          echo "TEST-UNEXPECTED-FAIL: $patch_type from $update_url wrong size"
+          echo "TEST-UNEXPECTED-FAIL: update.xml size: $size"
+          echo "TEST-UNEXPECTED-FAIL: actual size: $actual_size"
           return 1
       fi
 
       if [ $actual_hash != $hashValue ]; then
-          echo "FAIL: $patch_type from $update_url wrong hash"
-          echo "FAIL: update.xml hash: $hashValue"
-          echo "FAIL: actual hash: $actual_hash"
+          echo "TEST-UNEXPECTED-FAIL: $patch_type from $update_url wrong hash"
+          echo "TEST-UNEXPECTED-FAIL: update.xml hash: $hashValue"
+          echo "TEST-UNEXPECTED-FAIL: actual hash: $actual_hash"
           return 1
       fi
 
       cp update/$patch_type.mar update/update.mar
       echo $actual_size > update/$patch_type.size
 
     done
 }
--- a/tools/update-verify/release/updates/verify.sh
+++ b/tools/update-verify/release/updates/verify.sh
@@ -147,17 +147,17 @@ do
             "${to_build_id}" "${to_app_version}" "${to_display_version}"
           err=$?
         else
           download_mars "${aus_server}/update/3/${update_path}/update.xml?force=1" ${patch_type} 0 \
             "${to_build_id}" "${to_app_version}" "${to_display_version}"
           err=$?
         fi
         if [ "$err" != "0" ]; then
-          echo "FAIL: [${release} ${locale} ${patch_type}] download_mars returned non-zero exit code: ${err}"
+          echo "TEST-UNEXPECTED-FAIL: [${release} ${locale} ${patch_type}] download_mars returned non-zero exit code: ${err}"
           continue
         fi
       else
         mkdir -p updates/${update_path}/complete
         mkdir -p updates/${update_path}/partial
         $retry wget -q -O ${patch_type} updates/${update_path}/${patch_type}/update.xml "${aus_server}/update/3/${update_path}/update.xml?force=1"
 
       fi
@@ -239,49 +239,49 @@ do
             continue
         fi
 
         from_path=`echo $from | sed "s/%locale%/${locale}/"`
         to_path=`echo $to | sed "s/%locale%/${locale}/"`
         download_builds "${ftp_server_from}${from_path}" "${ftp_server_to}${to_path}"
         err=$?
         if [ "$err" != "0" ]; then
-          echo "FAIL: [$release $locale $patch_type] download_builds returned non-zero exit code: $err"
+          echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] download_builds returned non-zero exit code: $err"
           continue
         fi
         source_file=`basename "$from_path"`
         target_file=`basename "$to_path"`
         diff_file="results.diff"
         if [ -e ${diff_file} ]; then
           rm ${diff_file}
         fi
         check_updates "${platform}" "downloads/${source_file}" "downloads/${target_file}" ${locale} ${use_old_updater} ${updater} ${diff_file} ${channel} ${mar_channel_IDs}
         err=$?
         if [ "$err" == "0" ]; then
           continue
         elif [ "$err" == "1" ]; then
-          echo "FAIL: [$release $locale $patch_type] check_updates returned failure for $platform downloads/$source_file vs. downloads/$target_file: $err"
+          echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] check_updates returned failure for $platform downloads/$source_file vs. downloads/$target_file: $err"
         elif [ "$err" == "2" ]; then
           echo "WARN: [$release $locale $patch_type] check_updates returned warning for $platform downloads/$source_file vs. downloads/$target_file: $err"
         else
-          echo "FAIL: [$release $locale $patch_type] check_updates returned unknown error for $platform downloads/$source_file vs. downloads/$target_file: $err"
+          echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] check_updates returned unknown error for $platform downloads/$source_file vs. downloads/$target_file: $err"
         fi
 
         if [ -s ${diff_file} ]; then
           echo "Found diffs for ${patch_type} update from ${aus_server}/update/3/${update_path}/update.xml?force=1" >> ${diff_summary_log}
           cat ${diff_file} >> ${diff_summary_log}
           echo "" >> ${diff_summary_log}
         fi
       fi
     done
     if [ -f update/partial.size ] && [ -f update/complete.size ]; then
         partial_size=`cat update/partial.size`
         complete_size=`cat update/complete.size`
         if [ $partial_size -gt $complete_size ]; then
-            echo "FAIL: [$release $locale $patch_type] partial updates are larger than complete updates"
+            echo "TEST-UNEXPECTED-FAIL: [$release $locale $patch_type] partial updates are larger than complete updates"
         elif [ $partial_size -eq $complete_size ]; then
             echo "WARN: [$release $locale $patch_type] partial updates are the same size as complete updates, this should only happen for major updates"
         else
             echo "SUCCESS: [$release $locale $patch_type] partial updates are smaller than complete updates, all is well in the universe"
         fi
     fi
   done
 done < $config_file
--- a/tools/update-verify/scripts/chunked-verify.sh
+++ b/tools/update-verify/scripts/chunked-verify.sh
@@ -45,13 +45,13 @@ set +x
 echo "Scanning log for failures and warnings"
 echo "--------------------------------------"
 
 # Test for a failure, note we are set -e.
 # Grep returns 0 on a match and 1 on no match
 # Testing for failures first is important because it's OK to to mark as failed
 # when there's failures+warnings, but not OK to mark as warnings in the same
 # situation.
-( ! grep 'FAIL:' $SCRIPTS_DIR/../verify_log.txt ) || print_failed_msg
+( ! grep 'TEST-UNEXPECTED-FAIL:' $SCRIPTS_DIR/../verify_log.txt ) || print_failed_msg
 ( ! grep 'WARN:' $SCRIPTS_DIR/../verify_log.txt ) || print_warning_msg
 
 echo "-------------------------"
 echo "All is well"