Bug 1630809 - only generate release-partner-repack tasks for enabled platforms r=aki
authorNick Thomas <nthomas@mozilla.com>
Thu, 27 Aug 2020 22:19:48 +0000
changeset 546697 7efb583e8d110017d99efa27cfd314a9cfe6d640
parent 546696 9e38cbdb7b9612fc975f84326ee4a809479f026e
child 546698 f11d861827cd3aa5ccedc555a67cef597e2a9d6a
push id37736
push userapavel@mozilla.com
push dateFri, 28 Aug 2020 15:31:26 +0000
treeherdermozilla-central@56166cae2e26 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1630809
milestone82.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1630809 - only generate release-partner-repack tasks for enabled platforms r=aki Prior to this patch the task graph would always include a release-partner-repack-<platform> task for all 6 platforms, regardless of what was specified in release_partner_config. This was particularly obvious in the off-cycle respin scenario when a single partner is repacked. By moving and reusing get_repack_ids_by_platform() it's easy to skip unneeded platforms. Differential Revision: https://phabricator.services.mozilla.com/D87724
taskcluster/taskgraph/transforms/chunk_partners.py
taskcluster/taskgraph/transforms/partner_repack.py
taskcluster/taskgraph/util/partners.py
--- a/taskcluster/taskgraph/transforms/chunk_partners.py
+++ b/taskcluster/taskgraph/transforms/chunk_partners.py
@@ -7,57 +7,42 @@ Chunk the partner repack tasks by subpar
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
 
 from mozbuild.chunkify import chunkify
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.partners import (
-    get_partner_config_by_kind,
-    locales_per_build_platform,
+    get_repack_ids_by_platform,
     apply_partner_priority,
 )
 
 transforms = TransformSequence()
 transforms.add(apply_partner_priority)
 
 
-def _get_repack_ids_by_platform(partner_configs, build_platform):
-    combinations = []
-    for partner, partner_config in partner_configs.items():
-        for sub_partner, cfg in partner_config.items():
-            if build_platform not in cfg.get("platforms", []):
-                continue
-            locales = locales_per_build_platform(build_platform, cfg.get('locales', []))
-            for locale in locales:
-                combinations.append("{}/{}/{}".format(partner, sub_partner, locale))
-    return sorted(combinations)
-
-
 @transforms.add
 def chunk_partners(config, jobs):
-    partner_configs = get_partner_config_by_kind(config, config.kind)
-
     for job in jobs:
         dep_job = job['primary-dependency']
         build_platform = dep_job.attributes["build_platform"]
         repack_id = dep_job.task.get('extra', {}).get('repack_id')
         repack_ids = dep_job.task.get('extra', {}).get('repack_ids')
         copy_repack_ids = job.pop('copy-repack-ids', False)
 
         if copy_repack_ids:
             assert repack_ids, "dep_job {} doesn't have repack_ids!".format(
                 dep_job.label
             )
             job.setdefault('extra', {})['repack_ids'] = repack_ids
             yield job
         # first downstream of the repack task, no chunking or fanout has been done yet
         elif not any([repack_id, repack_ids]):
-            platform_repack_ids = _get_repack_ids_by_platform(partner_configs, build_platform)
+            platform_repack_ids = get_repack_ids_by_platform(config, build_platform)
             # we chunk mac signing
             if config.kind in ("release-partner-repack-signing",
                                "release-eme-free-repack-signing",
                                "release-partner-repack-notarization-part-1",
                                "release-eme-free-repack-notarization-part-1"):
                 repacks_per_chunk = job.get('repacks-per-chunk')
                 chunks, remainder = divmod(len(platform_repack_ids), repacks_per_chunk)
                 if remainder:
--- a/taskcluster/taskgraph/transforms/partner_repack.py
+++ b/taskcluster/taskgraph/transforms/partner_repack.py
@@ -8,25 +8,37 @@ Transform the partner repack task into a
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import resolve_keyed_by
 from taskgraph.util.scriptworker import get_release_config
 from taskgraph.util.partners import (
     check_if_partners_enabled,
     get_partner_url_config,
+    get_repack_ids_by_platform,
     apply_partner_priority,
 )
 
 
 transforms = TransformSequence()
 transforms.add(apply_partner_priority)
 
 
 @transforms.add
+def skip_unnecessary_platforms(config, tasks):
+    for task in tasks:
+        if config.kind == "release-partner-repack":
+            platform = task['attributes']['build_platform']
+            repack_ids = get_repack_ids_by_platform(config, platform)
+            if not repack_ids:
+                continue
+        yield task
+
+
+@transforms.add
 def populate_repack_manifests_url(config, tasks):
     for task in tasks:
         partner_url_config = get_partner_url_config(config.params, config.graph_config)
 
         for k in partner_url_config:
             if config.kind.startswith(k):
                 task['worker'].setdefault('env', {})['REPACK_MANIFESTS_URL'] = \
                     partner_url_config[k]
--- a/taskcluster/taskgraph/util/partners.py
+++ b/taskcluster/taskgraph/util/partners.py
@@ -405,16 +405,29 @@ def get_partner_url_config(parameters, g
     }
     resolve_keyed_by(partner_url_config, 'release-eme-free-repack', 'eme-free manifest_url',
                      **substitutions)
     resolve_keyed_by(partner_url_config, 'release-partner-repack', 'partner manifest url',
                      **substitutions)
     return partner_url_config
 
 
+def get_repack_ids_by_platform(config, build_platform):
+    partner_config = get_partner_config_by_kind(config, config.kind)
+    combinations = []
+    for partner, subconfigs in partner_config.items():
+        for sub_config_name, sub_config in subconfigs.items():
+            if build_platform not in sub_config.get("platforms", []):
+                continue
+            locales = locales_per_build_platform(build_platform, sub_config.get('locales', []))
+            for locale in locales:
+                combinations.append("{}/{}/{}".format(partner, sub_config_name, locale))
+    return sorted(combinations)
+
+
 def get_partners_to_be_published(config):
     # hardcoded kind because release-bouncer-aliases doesn't match otherwise
     partner_config = get_partner_config_by_kind(config, 'release-partner-repack')
     partners = []
     for partner, subconfigs in partner_config.items():
         for sub_config_name, sub_config in subconfigs.items():
             if sub_config.get("publish_to_releases"):
                 partners.append((partner, sub_config_name, sub_config['platforms']))