Bug 1648664 - Reduce download concurrency in partials. r=aki, a=release
authorSimon Fraser <sfraser@mozilla.com>
Tue, 30 Jun 2020 16:34:06 +0000
changeset 601876 817bf072a4864346b9903a64ad778559cb1e21ec
parent 601875 39740d9106a3cbcf9b4b106e4ebe4153dd073ff7
child 601877 49f59268153f16c3d4f12ac7e88109c14c0008ee
push id13337
push userryanvm@gmail.com
push dateWed, 01 Jul 2020 18:37:50 +0000
treeherdermozilla-beta@ecf6eea8a56f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki, release
bugs1648664
milestone79.0
Bug 1648664 - Reduce download concurrency in partials. r=aki, a=release Differential Revision: https://phabricator.services.mozilla.com/D81692
taskcluster/docker/funsize-update-generator/Makefile
taskcluster/docker/funsize-update-generator/README
taskcluster/docker/funsize-update-generator/runme.sh
taskcluster/docker/funsize-update-generator/scripts/funsize.py
--- a/taskcluster/docker/funsize-update-generator/Makefile
+++ b/taskcluster/docker/funsize-update-generator/Makefile
@@ -1,17 +1,9 @@
-DOCKERIO_USERNAME =$(error DOCKERIO_USERNAME should be set)
 IMAGE_NAME = funsize-update-generator
-FULL_IMAGE_NAME = $(DOCKERIO_USERNAME)/$(IMAGE_NAME)
 
 build:
-	docker build -t $(FULL_IMAGE_NAME) --no-cache --rm .
-
-push:
-	docker push $(FULL_IMAGE_NAME):latest
-
-pull:
-	docker pull $(FULL_IMAGE_NAME):latest
+	docker build -t $(IMAGE_NAME) --no-cache --rm .
 
 update_pubkeys:
 	curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/nightly_aurora_level3_primary.der | openssl x509 -inform DER -pubkey -noout > nightly.pubkey
 	curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/dep1.der | openssl x509 -inform DER -pubkey -noout > dep.pubkey
 	curl https://hg.mozilla.org/mozilla-central/raw-file/default/toolkit/mozapps/update/updater/release_primary.der | openssl x509 -inform DER -pubkey -noout > release.pubkey
--- a/taskcluster/docker/funsize-update-generator/README
+++ b/taskcluster/docker/funsize-update-generator/README
@@ -1,8 +1,7 @@
 
 To run this locally for testing/development purposes:
 
 1. Find a funsize generating task ID
-2. make pull DOCKERIO_USERNAME=mozillareleases
-3. docker run -t -e TASKCLUSTER_ROOT_URL="https://firefox-ci-tc.services.mozilla.com/" -e SHA1_SIGNING_CERT='nightly_sha1' -e SHA384_SIGNING_CERT='nightly_sha384' -e TASK_ID="${TASK_ID}" -e EXTRA_PARAMS="--arch=x86_64"  mozillareleases/funsize-update-generator /runme.sh
+2. docker run -t -e TASKCLUSTER_ROOT_URL="https://firefox-ci-tc.services.mozilla.com" -e SIGNING_CERT='nightly' -e MAR_CHANNEL_ID='firefox-mozilla-central' -e TASK_ID="${TASK_ID}" -e EXTRA_PARAMS="--arch=x86_64"  funsize-update-generator /runme.sh
 
 The TASK_ID should be a recent "partials" Task.
--- a/taskcluster/docker/funsize-update-generator/runme.sh
+++ b/taskcluster/docker/funsize-update-generator/runme.sh
@@ -3,16 +3,20 @@
 set -xe
 
 test "$TASK_ID"
 test "$SIGNING_CERT"
 
 ARTIFACTS_DIR="/home/worker/artifacts"
 mkdir -p "$ARTIFACTS_DIR"
 
+# Strip trailing / if present
+TASKCLUSTER_ROOT_URL="${TASKCLUSTER_ROOT_URL%/}"
+export TASKCLUSTER_ROOT_URL
+
 # duplicate the functionality of taskcluster-lib-urls, but in bash..
 queue_base="${TASKCLUSTER_ROOT_URL%/}/api/queue/v1"
 
 curl --location --retry 10 --retry-delay 10 -o /home/worker/task.json "$queue_base/task/$TASK_ID"
 
 # auth:aws-s3:read-write:tc-gp-private-1d-us-east-1/releng/mbsdiff-cache/
 # -> bucket of tc-gp-private-1d-us-east-1, path of releng/mbsdiff-cache/
 # Trailing slash is important, due to prefix permissions in S3.
--- a/taskcluster/docker/funsize-update-generator/scripts/funsize.py
+++ b/taskcluster/docker/funsize-update-generator/scripts/funsize.py
@@ -10,16 +10,17 @@ import asyncio
 import configparser
 import json
 import logging
 import os
 import shutil
 import tempfile
 import time
 from distutils.util import strtobool
+from contextlib import AsyncExitStack
 from pathlib import Path
 
 import aiohttp
 from mardor.reader import MarReader
 from mardor.signing import get_keysize
 from scriptworker.utils import retry_async, get_hash
 
 log = logging.getLogger(__name__)
@@ -59,19 +60,17 @@ def verify_signature(mar, cert):
             raise ValueError(
                 "MAR Signature invalid: %s (%s) against %s", mar, m.signature_type, cert
             )
 
 
 def process_arguments():
     parser = argparse.ArgumentParser()
     parser.add_argument("--artifacts-dir", required=True)
-    parser.add_argument(
-        "--signing-cert", type=argparse.FileType("rb"), required=True
-    )
+    parser.add_argument("--signing-cert", type=argparse.FileType("rb"), required=True)
     parser.add_argument("--task-definition", required=True, type=argparse.FileType("r"))
     parser.add_argument(
         "--allow-staging-prefixes",
         action="store_true",
         default=strtobool(os.environ.get("FUNSIZE_ALLOW_STAGING_PREFIXES", "false")),
         help="Allow files from staging buckets.",
     )
     parser.add_argument(
@@ -108,24 +107,27 @@ def validate_mar_channel_id(mar, channel
     if not found_channel_ids.issubset(channel_ids):
         raise ValueError(
             "MAR_CHANNEL_ID mismatch, {} not in {}".format(product_info[1], channel_ids)
         )
 
     log.info("%s channel %s in %s", mar, product_info[1], channel_ids)
 
 
-async def retry_download(*args, **kwargs):  # noqa: E999
+async def retry_download(*args, semaphore=None, **kwargs):  # noqa: E999
     """Retry download() calls."""
-    await retry_async(
-        download,
-        retry_exceptions=(aiohttp.ClientError, asyncio.TimeoutError),
-        args=args,
-        kwargs=kwargs,
-    )
+    async with AsyncExitStack() as stack:
+        if semaphore:
+            stack.enter_async_context(semaphore)
+        await retry_async(
+            download,
+            retry_exceptions=(aiohttp.ClientError, asyncio.TimeoutError),
+            args=args,
+            kwargs=kwargs,
+        )
 
 
 def verify_allowed_url(mar, allowed_url_prefixes):
     if not any(mar.startswith(prefix) for prefix in allowed_url_prefixes):
         raise ValueError(
             "{mar} is not in allowed URL prefixes: {p}".format(
                 mar=mar, p=allowed_url_prefixes
             )
@@ -225,31 +227,33 @@ def get_option(directory, filename, sect
 def extract_download_urls(partials_config, mar_type):
     """Extract a set of urls to download from the task configuration.
 
     mar_type should be one of "from", "to"
     """
     return {definition[f"{mar_type}_mar"] for definition in partials_config}
 
 
-async def download_and_verify_mars(
-    partials_config, allowed_url_prefixes, signing_cert
-):
+async def download_and_verify_mars(partials_config, allowed_url_prefixes, signing_cert):
     """Download, check signature, channel ID and unpack MAR files."""
     # Separate these categories so we can opt to perform checks on only 'to' downloads.
     from_urls = extract_download_urls(partials_config, mar_type="from")
     to_urls = extract_download_urls(partials_config, mar_type="to")
     tasks = list()
     downloads = dict()
+
+    semaphore = asyncio.Semaphore(2)  # Magic 2 to reduce network timeout errors.
     for url in from_urls.union(to_urls):
         verify_allowed_url(url, allowed_url_prefixes)
         downloads[url] = {
             "download_path": Path(tempfile.mkdtemp()) / Path(url).name,
         }
-        tasks.append(retry_download(url, downloads[url]["download_path"]))
+        tasks.append(
+            retry_download(url, downloads[url]["download_path"], semaphore=semaphore)
+        )
 
     await asyncio.gather(*tasks)
 
     for url in downloads:
         # Verify signature, but not from an artifact as we don't
         # depend on the signing task
         if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION") and not url.startswith(
             QUEUE_PREFIX