Bug 1266624: Apply exponential backoff for mozharness download. r=dustin
authorWander Lairson Costa <wcosta@mozilla.com>
Tue, 06 Dec 2016 15:27:40 -1000
changeset 325311 373caf284bbc8e8d9cff6638b11006d55b3cc1c0
parent 325310 ff3a2aebe47d050ba5bbef5cf2cc9848c49dc821
child 325312 b44c57f045c25e3af4e868e9a5ba76aa6060d275
child 325343 5ae2260e20e758d04e80427d29cf8aa1d2cd2890
push id24
push usermaklebus@msu.edu
push dateTue, 20 Dec 2016 03:11:33 +0000
reviewersdustin
bugs1266624
milestone53.0a1
Bug 1266624: Apply exponential backoff for mozharness download. r=dustin MozReview-Commit-ID: Ip4MjVJFwT6
taskcluster/scripts/tester/test-macosx.sh
taskcluster/scripts/tester/test-ubuntu.sh
--- a/taskcluster/scripts/tester/test-macosx.sh
+++ b/taskcluster/scripts/tester/test-macosx.sh
@@ -20,22 +20,48 @@ cd $WORKSPACE
 rm -rf artifacts
 mkdir artifacts
 
 # test required parameters are supplied
 if [[ -z ${MOZHARNESS_URL} ]]; then fail "MOZHARNESS_URL is not set"; fi
 if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi
 if [[ -z ${MOZHARNESS_CONFIG} ]]; then fail "MOZHARNESS_CONFIG is not set"; fi
 
-# Unzip the mozharness ZIP file created by the build task
-if ! curl --fail -o mozharness.zip --retry 10 -L $MOZHARNESS_URL; then
-    fail "failed to download mozharness zip"
-fi
-rm -rf mozharness
-unzip -q mozharness.zip
+# Download mozharness with exponential backoff
+# curl already applies exponential backoff, but not for all
+# failed cases, apparently, as we keep getting failed downloads
+# with 404 code.
+download_mozharness() {
+    local max_attempts=10
+    local timeout=1
+    local attempt=0
+
+    echo "Downloading mozharness"
+
+    while [[ $attempt < $max_attempts ]]; do
+        if curl --fail -o mozharness.zip --retry 10 -L $MOZHARNESS_URL; then
+            rm -rf mozharness
+            if unzip -q mozharness.zip; then
+                break
+            else
+                echo "error unzipping mozharness.zip" >&2
+            fi
+        else
+            echo "failed to download mozharness zip" >&2
+        fi
+        echo "Download failed, retrying in $timeout seconds..." >&2
+        sleep $timeout
+        timeout=$((timeout*2))
+        attempt=$((attempt+1))
+    done
+
+    fail "Failed to download and unzip mozharness"
+}
+
+download_mozharness
 rm mozharness.zip
 
 # For telemetry purposes, the build process wants information about the
 # source it is running; tc-vcs obscures this a little, but we can provide
 # it directly.
 export MOZ_SOURCE_REPO="${GECKO_HEAD_REPOSITORY}"
 export MOZ_SOURCE_CHANGESET="${GECKO_HEAD_REV}"
 
--- a/taskcluster/scripts/tester/test-ubuntu.sh
+++ b/taskcluster/scripts/tester/test-ubuntu.sh
@@ -64,25 +64,49 @@ cleanup() {
       # To share X issues
       cp /home/worker/.xsession-errors ~/artifacts/public/xsession-errors.log
     fi
     cleanup_xvfb
     exit $rv
 }
 trap cleanup EXIT INT
 
+# Download mozharness with exponential backoff
+# curl already applies exponential backoff, but not for all
+# failed cases, apparently, as we keep getting failed downloads
+# with 404 code.
+download_mozharness() {
+    local max_attempts=10
+    local timeout=1
+    local attempt=0
+
+    echo "Downloading mozharness"
+
+    while [[ $attempt < $max_attempts ]]; do
+        if curl --fail -o mozharness.zip --retry 10 -L $MOZHARNESS_URL; then
+            rm -rf mozharness
+            if unzip -q mozharness.zip; then
+                return 0
+            fi
+            echo "error unzipping mozharness.zip" >&2
+        else
+            echo "failed to download mozharness zip" >&2
+        fi
+        echo "Download failed, retrying in $timeout seconds..." >&2
+        sleep $timeout
+        timeout=$((timeout*2))
+        attempt=$((attempt+1))
+    done
+
+    fail "Failed to download and unzip mozharness"
+}
+
 # Download mozharness if we're told to.
 if [ ${MOZHARNESS_URL} ]; then
-    if ! curl --fail -o mozharness.zip --retry 10 -L $MOZHARNESS_URL; then
-        fail "failed to download mozharness zip"
-    fi
-    rm -rf mozharness
-    if !  unzip -q mozharness.zip; then
-        fail "error unzipping mozharness.zip"
-    fi
+    download_mozharness
     rm mozharness.zip
 
     if ! [ -d mozharness ]; then
         fail "mozharness zip did not contain mozharness/"
     fi
 
     MOZHARNESS_PATH=`pwd`/mozharness
 fi