Bug 1171527 - Make upload_symbols.py retry on 500 errors from the API. r=gps, a=NPOTB
authorTed Mielczarek <ted@mielczarek.org>
Fri, 05 Jun 2015 07:38:14 -0400
changeset 266255 04059bd01b9b
parent 266254 acaf3ba8f4e4
child 266256 68049b6deb7c
push id4804
push userryanvm@gmail.com
push date2015-06-15 15:02 +0000
treeherdermozilla-beta@05b522f50491 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps, NPOTB
bugs1171527
milestone39.0
Bug 1171527 - Make upload_symbols.py retry on 500 errors from the API. r=gps, a=NPOTB
toolkit/crashreporter/tools/upload_symbols.py
--- a/toolkit/crashreporter/tools/upload_symbols.py
+++ b/toolkit/crashreporter/tools/upload_symbols.py
@@ -17,16 +17,32 @@ from __future__ import print_function
 import os
 import redo
 import requests
 import sys
 
 from buildconfig import substs
 
 url = 'https://crash-stats.mozilla.com/symbols/upload'
+MAX_RETRIES = 5
+
+def print_error(r):
+    if r.status_code < 400:
+        print('Error: bad auth token? ({0}: {1})'.format(r.status_code,
+                                                         r.reason),
+              file=sys.stderr)
+    else:
+        print('Error: got HTTP response {0}: {1}'.format(r.status_code,
+                                                         r.reason),
+              file=sys.stderr)
+
+    print('Response body:\n{sep}\n{body}\n{sep}\n'.format(
+        sep='=' * 20,
+        body=r.text
+        ))
 
 def main():
     if len(sys.argv) != 2:
         print('Usage: uploadsymbols.py <zip file>', file=sys.stderr)
         return 1
 
     if not os.path.isfile(sys.argv[1]):
         print('Error: zip file "{0}" does not exist!'.format(sys.argv[1]),
@@ -39,47 +55,41 @@ def main():
         return 1
     token_file = substs['SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE']
 
     if not os.path.isfile(token_file):
         print('Error: SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "{0}" does not exist!'.format(token_file), file=sys.stderr)
         return 1
     auth_token = open(token_file, 'r').read().strip()
 
-    print('Uploading symbol file "{0}" to "{1}"...'.format(sys.argv[1], url))
+    print('Uploading symbol file "{0}" to "{1}"'.format(sys.argv[1], url))
 
-    for _ in redo.retrier():
+    for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
+        print('Attempt %d of %d...' % (i, MAX_RETRIES))
         try:
             r = requests.post(
                 url,
                 files={'symbols.zip': open(sys.argv[1], 'rb')},
                 headers={'Auth-Token': auth_token},
                 allow_redirects=False,
                 timeout=120)
-            break
+            # 500 is likely to be a transient failure.
+            # Break out for success or other error codes.
+            if r.status_code  != 500:
+                break
+            print_error(r)
         except requests.exceptions.RequestException as e:
             print('Error: {0}'.format(e))
+        print('Retrying...')
     else:
         print('Maximum retries hit, giving up!')
         return 1
 
     if r.status_code >= 200 and r.status_code < 300:
         print('Uploaded successfully!')
         return 0
 
-    if r.status_code < 400:
-        print('Error: bad auth token? ({0}: {1})'.format(r.status_code,
-                                                         r.reason),
-              file=sys.stderr)
-    else:
-        print('Error: got HTTP response {0}: {1}'.format(r.status_code,
-                                                         r.reason),
-              file=sys.stderr)
-
-    print('Response body:\n{sep}\n{body}\n{sep}\n'.format(
-        sep='=' * 20,
-        body=r.text
-        ))
+    print_error(r)
     return 1
 
 if __name__ == '__main__':
     sys.exit(main())