Bug 1523324 - Add retries to upload_generated_sources.py; r=nalexander, a=release
authorMike Shal <mshal@mozilla.com>
Fri, 03 May 2019 23:26:51 +0000
changeset 526496 a78e6aced633fc0f1ca9718af1f044a0b2274b24
parent 526495 d3c4aaf58b19562a3d291dbd020895eb3dd2c141
child 526497 d6c30fa0365bc13b59d1dd58b34171872e2b51b5
push id2032
push userffxbld-merge
push dateMon, 13 May 2019 09:36:57 +0000
treeherdermozilla-release@455c1065dcbe [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander, release
bugs1523324
milestone67.0
Bug 1523324 - Add retries to upload_generated_sources.py; r=nalexander, a=release Occasionally the taskcluster fetches can fail, so make sure the requests library automatically retries if possible. Differential Revision: https://phabricator.services.mozilla.com/D29914
build/upload_generated_sources.py
--- a/build/upload_generated_sources.py
+++ b/build/upload_generated_sources.py
@@ -15,16 +15,17 @@ from mozbuild.generated_sources import (
     get_filename_with_digest,
     get_s3_region_and_bucket,
 )
 import os
 from Queue import Queue
 import requests
 import sys
 import tarfile
+from requests.packages.urllib3.util.retry import Retry
 from threading import Event, Thread
 import time
 
 # Arbitrary, should probably measure this.
 NUM_WORKER_THREADS = 10
 log = logging.getLogger('upload-generated-sources')
 log.setLevel(logging.INFO)
 
@@ -87,16 +88,22 @@ def upload_worker(queue, event, bucket, 
     except Exception:
         log.exception('Thread encountered exception:')
         event.set()
 
 
 def do_work(artifact, region, bucket):
     session_args = {'region_name': region}
     session = requests.Session()
+    retry = Retry(total=5, backoff_factor=0.1,
+                  status_forcelist=[500, 502, 503, 504])
+    http_adapter = requests.adapters.HTTPAdapter(max_retries=retry)
+    session.mount('https://', http_adapter)
+    session.mount('http://', http_adapter)
+
     if 'TASK_ID' in os.environ:
         level = os.environ.get('MOZ_SCM_LEVEL', '1')
         secrets_url = 'http://taskcluster/secrets/v1/secret/project/releng/gecko/build/level-{}/gecko-generated-sources-upload'.format( # noqa
             level)
         log.info(
             'Using AWS credentials from the secrets service: "{}"'.format(secrets_url))
         res = session.get(secrets_url)
         res.raise_for_status()