Bug 1430600 - Fix retries for partials async downloads. r=callek, a=release
authorSimon Fraser <sfraser@mozilla.com>
Tue, 03 Apr 2018 18:31:05 +0100
changeset 463050 91076da795a15d79657b6120a72f2816fc2dc427
parent 463049 efefd8841141dcf1ff5c628c3a22f73686d8711f
child 463051 88955ced286b9353702b5e2deb3268cd8c9fe772
push id1683
push usersfraser@mozilla.com
push dateThu, 26 Apr 2018 16:43:40 +0000
treeherdermozilla-release@5af6cb21869d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerscallek, release
bugs1430600
milestone60.0
Bug 1430600 - Fix retries for partials async downloads. r=callek, a=release Summary: This doesn't solve the download timeouts, but does ensure that retries are happening as they should. Reviewers: rail Reviewed By: rail Bug #: 1430600 Differential Revision: https://phabricator.services.mozilla.com/D821
taskcluster/docker/funsize-update-generator/scripts/funsize.py
--- a/taskcluster/docker/funsize-update-generator/scripts/funsize.py
+++ b/taskcluster/docker/funsize-update-generator/scripts/funsize.py
@@ -92,44 +92,43 @@ def get_secret(secret_name):
     return r.json().get('secret', {})
 
 
 async def retry_download(*args, **kwargs):  # noqa: E999
     """Retry download() calls."""
     await retry_async(
         download,
         retry_exceptions=(
-            aiohttp.ClientError
+            aiohttp.ClientError,
+            asyncio.TimeoutError
         ),
         args=args,
         kwargs=kwargs
     )
 
 
 async def download(url, dest, mode=None):  # noqa: E999
     log.info("Downloading %s to %s", url, dest)
 
     bytes_downloaded = 0
-
     async with aiohttp.ClientSession(raise_for_status=True) as session:
-        async with session.get(url) as resp:
+        async with session.get(url, timeout=60) as resp:
             with open(dest, 'wb') as fd:
                 while True:
                     chunk = await resp.content.read(4096)
                     if not chunk:
                         break
                     fd.write(chunk)
                     bytes_downloaded += len(chunk)
 
             log.debug('Downloaded %s bytes', bytes_downloaded)
             if 'content-length' in resp.headers:
                 log.debug('Content-Length: %s bytes', resp.headers['content-length'])
                 if bytes_downloaded != int(resp.headers['content-length']):
                     raise IOError('Unexpected number of bytes downloaded')
-
             if mode:
                 log.debug("chmod %o %s", mode, dest)
                 os.chmod(dest, mode)
 
 
 async def run_command(cmd, cwd='/', env=None, label=None, silent=False):
     if not env:
         env = dict()