Bug 1501141 - restore two files for final verify
authorNick Thomas <nthomas@mozilla.com>
Tue, 23 Oct 2018 15:42:42 +1300
changeset 8462 f48e358fe285b8973c6300d6592e27d206300d47
parent 8461 482c0f6537d47f7cbf1326066ecd1d3efbbd53a7
child 8463 5403bbf3cffa1a1b368c7119faf5421f1d088e5f
push id6185
push usernthomas@mozilla.com
push dateTue, 23 Oct 2018 02:42:55 +0000
bugs1501141
Bug 1501141 - restore two files for final verify
release/get-update-xml.sh
release/test-mar-url.sh
new file mode 100755
--- /dev/null
+++ b/release/get-update-xml.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+update_xml_url="${1}"
+patch_types="${2}"
+update_xml="$(mktemp -t update.xml.XXXXXXXXXX)"
+update_xml_headers="$(mktemp -t update.xml.headers.XXXXXXXXXX)"
+update_xml_debug="$(mktemp -t update.xml.debug.XXXXXXXXXX)"
+curl --retry 50 --retry-max-time 300 -k -s -D "${update_xml_headers}" -L -v "${update_xml_url}" > "${update_xml}" 2>"${update_xml_debug}"
+update_xml_curl_exit_code=$?
+if [ "${update_xml_curl_exit_code}" == 0 ]
+then
+    update_xml_actual_url="$(sed -e "s/$(printf '\r')//" -n -e 's/^Location: //p' "${update_xml_headers}" | tail -1)"
+    [ -n "${update_xml_actual_url}" ] && update_xml_url_with_redirects="${update_xml_url} => ${update_xml_actual_url}" || update_xml_url_with_redirects="${update_xml_url}"
+    echo "$(date):  Downloaded update.xml file from ${update_xml_url_with_redirects}" > "$(mktemp -t log.XXXXXXXXXX)"
+    for patch_type in ${patch_types//,/ }
+    do  
+        mar_url_and_size="$(sed -e 's/\&amp;/\&/g' -n -e 's/.*<patch .*type="'"${patch_type}"'".* URL="\([^"]*\)".*size="\([^"]*\)".*/\1 \2/p' "${update_xml}" | tail -1)"
+        if [ -z "${mar_url_and_size}" ]
+        then
+            echo "$(date):  FAILURE: No patch type '${patch_type}' found in update.xml from ${update_xml_url_with_redirects}" > "$(mktemp -t log.XXXXXXXXXX)"
+            echo "PATCH_TYPE_MISSING ${update_xml_url} ${patch_type} ${update_xml} ${update_xml_headers} ${update_xml_debug} ${update_xml_actual_url}" > "$(mktemp -t failure.XXXXXXXXXX)"
+        else
+            echo "$(date):  Mar url and file size for patch type '${patch_type}' extracted from ${update_xml_url_with_redirects} (${mar_url_and_size})" > "$(mktemp -t log.XXXXXXXXXX)"
+            echo "${mar_url_and_size} ${update_xml_url} ${patch_type} ${update_xml_actual_url}" > "$(mktemp -t update_xml_to_mar.XXXXXXXXXX)"
+        fi
+    done
+else
+    if [ -z "${update_xml_actual_url}" ]
+    then
+        echo "$(date):  FAILURE: Could not retrieve update.xml from ${update_xml_url} for patch type(s) '${patch_types}'" > "$(mktemp -t log.XXXXXXXXXX)"
+        echo "UPDATE_XML_UNAVAILABLE ${update_xml_url} ${update_xml} ${update_xml_headers} ${update_xml_debug} ${update_xml_curl_exit_code}" > "$(mktemp -t failure.XXXXXXXXXX)"
+    else
+        echo "$(date):  FAILURE: update.xml from ${update_xml_url} redirected to ${update_xml_actual_url} but could not retrieve update.xml from here" > "$(mktemp -t log.XXXXXXXXXX)"
+        echo "UPDATE_XML_REDIRECT_FAILED ${update_xml_url} ${update_xml_actual_url} ${update_xml} ${update_xml_headers} ${update_xml_debug} ${update_xml_curl_exit_code}" > "$(mktemp -t failure.XXXXXXXXXX)"
+    fi
+fi
new file mode 100755
--- /dev/null
+++ b/release/test-mar-url.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+mar_url="${1}"
+mar_required_size="${2}"
+
+mar_headers_file="$(mktemp -t mar_headers.XXXXXXXXXX)"
+mar_headers_debug_file="$(mktemp -t mar_headers_debug.XXXXXXXXXX)"
+curl --retry 50 --retry-max-time 300 -k -s -i -r 0-2 -L -v "${mar_url}" > "${mar_headers_file}" 2>"${mar_headers_debug_file}"
+mar_file_curl_exit_code=$?
+
+# Bug 894368 - HTTP 408's are not handled by the "curl --retry" mechanism; in this case retry in bash
+attempts=1
+while [ "$((++attempts))" -lt 50 ] && grep 'HTTP/1\.1 408 Request Timeout' "${mar_headers_file}" &>/dev/null
+do
+    sleep 1
+    curl --retry 50 --retry-max-time 300 -k -s -i -r 0-2 -L -v "${mar_url}" > "${mar_headers_file}" 2>"${mar_headers_debug_file}"
+    mar_file_curl_exit_code=$?
+done
+
+# check file size matches what was written in update.xml
+# strip out dos line returns from header if they occur
+# note: below, using $(printf '\r') for Darwin compatibility, rather than simple '\r'
+# (i.e. shell interprets '\r' rather than sed interpretting '\r')
+mar_actual_size="$(sed -e "s/$(printf '\r')//" -n -e 's/^Content-Range: bytes 0-2\///ip' "${mar_headers_file}" | tail -1)"
+mar_actual_url="$(sed -e "s/$(printf '\r')//" -n -e 's/^Location: //p' "${mar_headers_file}" | tail -1)"
+# note: below, sed -n '/^HTTP\//p' acts as grep '^HTTP/', but requires less overhead as sed already running
+http_response_code="$(sed -e "s/$(printf '\r')//" -n -e '/^HTTP\//p' "${mar_headers_file}" | tail -1)"
+
+[ -n "${mar_actual_url}" ] && mar_url_with_redirects="${mar_url} => ${mar_actual_url}" || mar_url_with_redirects="${mar_url}"
+
+if [ "${mar_actual_size}" == "${mar_required_size}" ]
+then
+    echo "$(date):  Mar file ${mar_url_with_redirects} available with correct size (${mar_actual_size} bytes)" > "$(mktemp -t log.XXXXXXXXXX)"
+elif [ -z "${mar_actual_size}" ]
+then
+    echo "$(date):  FAILURE: Could not retrieve http header for mar file from ${mar_url}" > "$(mktemp -t log.XXXXXXXXXX)"
+    echo "NO_MAR_FILE ${mar_url} ${mar_headers_file} ${mar_headers_debug_file} ${mar_file_curl_exit_code} ${mar_actual_url}" > "$(mktemp -t failure.XXXXXXXXXX)"
+    # If we get a response code (i.e. not an empty string), it better contain "206 Partial Content" or we should report on it.
+    # If response code is empty, this should be caught by a different block to this one (e.g. "could not retrieve http header").
+elif [ -n "${http_response_code}" ] && [ "${http_response_code}" == "${http_response_code/206 Partial Content/}" ]
+then
+    echo "$(date):  FAILURE: received a '${http_response_code}' response for mar file from ${mar_url} (expected HTTP 206 Partial Content)" > "$(mktemp -t log.XXXXXXXXXX)"
+    echo "BAD_HTTP_RESPONSE_CODE_FOR_MAR ${mar_url} ${mar_headers_file} ${mar_headers_debug_file} ${mar_file_curl_exit_code} ${mar_actual_url}" > "$(mktemp -t failure.XXXXXXXXXX)"
+else
+    echo "$(date):  FAILURE: Mar file incorrect size - should be ${mar_required_size} bytes, but is ${mar_actual_size} bytes - ${mar_url_with_redirects}" > "$(mktemp -t log.XXXXXXXXXX)"
+    echo "MAR_FILE_WRONG_SIZE ${mar_url} ${mar_required_size} ${mar_actual_size} ${mar_headers_file} ${mar_headers_debug_file} ${mar_file_curl_exit_code} ${mar_actual_url}" > "$(mktemp -t failure.XXXXXXXXXX)"
+fi