Bug 1171527 - Make upload_symbols.py retry on 500 errors from the API. r=gps, a=NPOTB
--- a/toolkit/crashreporter/tools/upload_symbols.py
+++ b/toolkit/crashreporter/tools/upload_symbols.py
@@ -17,16 +17,32 @@ from __future__ import print_function
import os
import redo
import requests
import sys
from buildconfig import substs
url = 'https://crash-stats.mozilla.com/symbols/upload'
+MAX_RETRIES = 5
+
+def print_error(r):
+ if r.status_code < 400:
+ print('Error: bad auth token? ({0}: {1})'.format(r.status_code,
+ r.reason),
+ file=sys.stderr)
+ else:
+ print('Error: got HTTP response {0}: {1}'.format(r.status_code,
+ r.reason),
+ file=sys.stderr)
+
+ print('Response body:\n{sep}\n{body}\n{sep}\n'.format(
+ sep='=' * 20,
+ body=r.text
+ ))
def main():
if len(sys.argv) != 2:
print('Usage: uploadsymbols.py <zip file>', file=sys.stderr)
return 1
if not os.path.isfile(sys.argv[1]):
print('Error: zip file "{0}" does not exist!'.format(sys.argv[1]),
@@ -39,47 +55,41 @@ def main():
return 1
token_file = substs['SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE']
if not os.path.isfile(token_file):
print('Error: SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "{0}" does not exist!'.format(token_file), file=sys.stderr)
return 1
auth_token = open(token_file, 'r').read().strip()
- print('Uploading symbol file "{0}" to "{1}"...'.format(sys.argv[1], url))
+ print('Uploading symbol file "{0}" to "{1}"'.format(sys.argv[1], url))
- for _ in redo.retrier():
+ for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
+ print('Attempt %d of %d...' % (i, MAX_RETRIES))
try:
r = requests.post(
url,
files={'symbols.zip': open(sys.argv[1], 'rb')},
headers={'Auth-Token': auth_token},
allow_redirects=False,
timeout=120)
- break
+ # 500 is likely to be a transient failure.
+ # Break out for success or other error codes.
+ if r.status_code != 500:
+ break
+ print_error(r)
except requests.exceptions.RequestException as e:
print('Error: {0}'.format(e))
+ print('Retrying...')
else:
print('Maximum retries hit, giving up!')
return 1
if r.status_code >= 200 and r.status_code < 300:
print('Uploaded successfully!')
return 0
- if r.status_code < 400:
- print('Error: bad auth token? ({0}: {1})'.format(r.status_code,
- r.reason),
- file=sys.stderr)
- else:
- print('Error: got HTTP response {0}: {1}'.format(r.status_code,
- r.reason),
- file=sys.stderr)
-
- print('Response body:\n{sep}\n{body}\n{sep}\n'.format(
- sep='=' * 20,
- body=r.text
- ))
+ print_error(r)
return 1
if __name__ == '__main__':
sys.exit(main())