Bug 1430600 Fix retries for partials async downloads r=callek
authorSimon Fraser <sfraser@mozilla.com>
Tue, 03 Apr 2018 18:31:05 +0100
changeset 776791 b57020e5e59b3448b0d232f77d3ea79f01a8ce11
parent 776790 7e3e50fe7d197df6cef86043c79bc80b88e3b223
child 776792 e4c4f0ee3e74bf899d1fee024abfd067b6ec2509
push id104994
push usermaglione.k@gmail.com
push dateTue, 03 Apr 2018 18:21:08 +0000
reviewerscallek
bugs1430600
milestone61.0a1
Bug 1430600 Fix retries for partials async downloads r=callek Summary: This doesn't solve the download timeouts, but does ensure that retries are happening as they should. Reviewers: rail Reviewed By: rail Bug #: 1430600 Differential Revision: https://phabricator.services.mozilla.com/D821
taskcluster/docker/funsize-update-generator/scripts/funsize.py
--- a/taskcluster/docker/funsize-update-generator/scripts/funsize.py
+++ b/taskcluster/docker/funsize-update-generator/scripts/funsize.py
@@ -92,44 +92,43 @@ def get_secret(secret_name):
     return r.json().get('secret', {})
 
 
 async def retry_download(*args, **kwargs):  # noqa: E999
     """Retry download() calls."""
     await retry_async(
         download,
         retry_exceptions=(
-            aiohttp.ClientError
+            aiohttp.ClientError,
+            asyncio.TimeoutError
         ),
         args=args,
         kwargs=kwargs
     )
 
 
 async def download(url, dest, mode=None):  # noqa: E999
     log.info("Downloading %s to %s", url, dest)
 
     bytes_downloaded = 0
-
     async with aiohttp.ClientSession(raise_for_status=True) as session:
-        async with session.get(url) as resp:
+        async with session.get(url, timeout=60) as resp:
             with open(dest, 'wb') as fd:
                 while True:
                     chunk = await resp.content.read(4096)
                     if not chunk:
                         break
                     fd.write(chunk)
                     bytes_downloaded += len(chunk)
 
             log.debug('Downloaded %s bytes', bytes_downloaded)
             if 'content-length' in resp.headers:
                 log.debug('Content-Length: %s bytes', resp.headers['content-length'])
                 if bytes_downloaded != int(resp.headers['content-length']):
                     raise IOError('Unexpected number of bytes downloaded')
-
             if mode:
                 log.debug("chmod %o %s", mode, dest)
                 os.chmod(dest, mode)
 
 
 async def run_command(cmd, cwd='/', env=None, label=None, silent=False):
     if not env:
         env = dict()