Bug 1523324 - Add retries to upload_generated_sources.py; r=nalexander
authorMike Shal <mshal@mozilla.com>
Fri, 03 May 2019 23:26:51 +0000
changeset 531429 4dabc47dec1223930955d6e7eeb09d7fb5a61661
parent 531428 ee580278f872fc2b68631da034d80a3dbf4eed2a
child 531430 09d50908a15e4e0010acae2eb1ca3e8e121455b2
push id11265
push userffxbld-merge
push dateMon, 13 May 2019 10:53:39 +0000
treeherdermozilla-beta@77e0fe8dbdd3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnalexander
bugs1523324
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1523324 - Add retries to upload_generated_sources.py; r=nalexander Occasionally the taskcluster fetches can fail, so make sure the requests library automatically retries if possible. Differential Revision: https://phabricator.services.mozilla.com/D29914
build/upload_generated_sources.py
--- a/build/upload_generated_sources.py
+++ b/build/upload_generated_sources.py
@@ -15,16 +15,17 @@ from mozbuild.generated_sources import (
     get_filename_with_digest,
     get_s3_region_and_bucket,
 )
 import os
 from Queue import Queue
 import requests
 import sys
 import tarfile
+from requests.packages.urllib3.util.retry import Retry
 from threading import Event, Thread
 import time
 
 # Arbitrary, should probably measure this.
 NUM_WORKER_THREADS = 10
 log = logging.getLogger('upload-generated-sources')
 log.setLevel(logging.INFO)
 
@@ -87,16 +88,22 @@ def upload_worker(queue, event, bucket, 
     except Exception:
         log.exception('Thread encountered exception:')
         event.set()
 
 
 def do_work(artifact, region, bucket):
     session_args = {'region_name': region}
     session = requests.Session()
+    retry = Retry(total=5, backoff_factor=0.1,
+                  status_forcelist=[500, 502, 503, 504])
+    http_adapter = requests.adapters.HTTPAdapter(max_retries=retry)
+    session.mount('https://', http_adapter)
+    session.mount('http://', http_adapter)
+
     if 'TASK_ID' in os.environ:
         level = os.environ.get('MOZ_SCM_LEVEL', '1')
         secrets_url = 'http://taskcluster/secrets/v1/secret/project/releng/gecko/build/level-{}/gecko-generated-sources-upload'.format( # noqa
             level)
         log.info(
             'Using AWS credentials from the secrets service: "{}"'.format(secrets_url))
         res = session.get(secrets_url)
         res.raise_for_status()