Bug 1538995 - submit partner products to bouncer when we'll host the files, r=aki
authorNick Thomas <nthomas@mozilla.com>
Fri, 21 Jun 2019 22:15:01 +0000
changeset 537292 86fac3055d1042916b24e3efb436505c75ad31fd
parent 537291 19d9e73e3c0e31be437c032c4b6d0d1bb8101959
child 537293 1f1987d0fef378d50f9d1c7b036ff9508ca17345
push id2128
push usernthomas@mozilla.com
push dateWed, 14 Aug 2019 02:49:25 +0000
treeherdermozilla-release@2254823049a2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1538995
milestone68.0.2
Bug 1538995 - submit partner products to bouncer when we'll host the files, r=aki Adds a new release-partner-repack-bouncer-sub-firefox task to add the partner products. This is forked from the main bouncer submission because it diverges sufficiently that this is a cleaner approach. The get_partners_to_be_published helper is used by aliases too. Depends on D34950 Differential Revision: https://phabricator.services.mozilla.com/D35481
taskcluster/ci/release-partner-repack-bouncer-sub/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/bouncer_submission_partners.py
taskcluster/taskgraph/util/partners.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-partner-repack-bouncer-sub/kind.yml
@@ -0,0 +1,45 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+transforms:
+    - taskgraph.transforms.release:run_on_releases
+    - taskgraph.transforms.bouncer_submission_partners:transforms
+    - taskgraph.transforms.task:transforms
+
+job-defaults:
+    description: release partners bouncer submission job
+    worker-type:
+        by-release-level:
+            production: scriptworker-prov-v1/bouncer-v1
+            staging: scriptworker-prov-v1/bouncer-dev
+    worker:
+        implementation: bouncer-submission
+    scopes:
+        by-release-level:
+            production:
+                - project:releng:bouncer:action:submission
+                - project:releng:bouncer:server:production
+            staging:
+                - project:releng:bouncer:action:submission
+                - project:releng:bouncer:server:staging
+    run-on-projects: []
+    shipping-phase: promote
+    locales-file: browser/locales/shipped-locales
+    treeherder:
+        symbol: Rel(BncSub-P)
+        kind: test
+        tier: 1
+
+jobs:
+    firefox:
+        bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64', 'win64-aarch64']
+        bouncer-products:
+            by-release-type:
+                default: ['installer', 'stub-installer']
+                esr.*: []
+        shipping-product: firefox
+        treeherder:
+            platform: firefox-release/opt
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -417,16 +417,21 @@ Repackaging of partner repacks.
 release-partner-repack-repackage-signing
 ----------------------------------------
 External signing of partner repacks.
 
 release-partner-repack-beetmover
 --------------------------------
 Moves the partner repacks to S3 buckets.
 
+release-partner-repack-bouncer-sub
+----------------------------------
+Sets up bouncer products for partners.
+
+
 release-early-tagging
 ---------------------
 Utilises treescript to perform tagging that should happen near the start of a release.
 
 release-eme-free-repack
 -----------------------
 Generates customized versions of releases for eme-free repacks.
 
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/bouncer_submission_partners.py
@@ -0,0 +1,165 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Add from parameters.yml into bouncer submission tasks.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.bouncer_submission import (
+    FTP_PLATFORMS_PER_BOUNCER_PLATFORM,
+    CONFIG_PER_BOUNCER_PRODUCT as CONFIG_PER_BOUNCER_PRODUCT_VANILLA,
+    _craft_ftp_product,
+    _craft_filename_product,
+)
+from taskgraph.util.partners import (
+    check_if_partners_enabled,
+    get_partners_to_be_published,
+)
+from taskgraph.util.schema import resolve_keyed_by
+from taskgraph.util.scriptworker import get_release_config
+
+logger = logging.getLogger(__name__)
+
+
+PARTNER_PLATFORMS_TO_BOUNCER = {
+    "linux-shippable": "linux",
+    "linux64-shippable": "linux64",
+    "macosx64-shippable": "osx",
+    "win32-shippable": "win",
+    "win64-shippable": "win64",
+    "win64-aarch64-shippable": "win64-aarch64"
+}
+
+# :lang is interpolated by bouncer at runtime
+RELEASES_PARTNERS_PATH_TEMPLATE = '/{ftp_product}/releases/partners/{partner}/{sub_config}/\
+{version}/{ftp_platform}/:lang/{file}'
+
+CONFIG_PER_BOUNCER_PRODUCT = {
+    'installer': {
+        'name_postfix': "-{partner}-{sub_config}",
+        'path_template': RELEASES_PARTNERS_PATH_TEMPLATE,
+        'file_names':  CONFIG_PER_BOUNCER_PRODUCT_VANILLA['installer']['file_names']
+    },
+    'stub-installer': {
+        'name_postfix': "-{partner}-{sub_config}-stub",
+        # We currently have a sole win32 stub installer that is to be used
+        # in all windows platforms to toggle between full installers
+        'path_template': RELEASES_PARTNERS_PATH_TEMPLATE.replace('{ftp_platform}', 'win32'),
+        'file_names': CONFIG_PER_BOUNCER_PRODUCT_VANILLA['stub-installer']['file_names']
+    }
+}
+
+transforms = TransformSequence()
+transforms.add(check_if_partners_enabled)
+
+
+@transforms.add
+def make_task_worker(config, jobs):
+    for job in jobs:
+        resolve_keyed_by(
+            job, 'worker-type', item_name=job['name'],
+            **{'release-level': config.params.release_level()}
+        )
+        resolve_keyed_by(
+            job, 'scopes', item_name=job['name'],
+            **{'release-level': config.params.release_level()}
+        )
+        resolve_keyed_by(
+            job, 'bouncer-products', item_name=job['name'],
+            **{'release-type': config.params['release_type']}
+        )
+
+        # the schema requires at least one locale but this will not be used
+        job['worker']['locales'] = ["fake"]
+        job['worker']['entries'] = craft_bouncer_entries(config, job)
+
+        del job['locales-file']
+        del job['bouncer-platforms']
+        del job['bouncer-products']
+
+        if job['worker']['entries']:
+            yield job
+
+
+def craft_bouncer_entries(config, job):
+    release_config = get_release_config(config)
+
+    product = job['shipping-product']
+    current_version = release_config['version']
+    bouncer_products = job['bouncer-products']
+
+    partners = get_partners_to_be_published(config)
+    entries = {}
+    for partner, sub_config_name, platforms in partners:
+        platforms = [PARTNER_PLATFORMS_TO_BOUNCER[p] for p in platforms]
+        entries.update({
+            craft_partner_bouncer_product_name(product, bouncer_product, current_version,
+                                               partner, sub_config_name): {
+                'options': {
+                    'add_locales': False,  # partners may use different sets of locales
+                    'ssl_only': craft_ssl_only(bouncer_product),
+                },
+                'paths_per_bouncer_platform': craft_paths_per_bouncer_platform(
+                    product, bouncer_product, platforms, current_version, partner,
+                    sub_config_name
+                ),
+            }
+            for bouncer_product in bouncer_products
+        })
+    return entries
+
+
+def craft_paths_per_bouncer_platform(product, bouncer_product, bouncer_platforms, current_version,
+                                     partner, sub_config):
+    paths_per_bouncer_platform = {}
+    for bouncer_platform in bouncer_platforms:
+        file_names_per_platform = CONFIG_PER_BOUNCER_PRODUCT[bouncer_product]['file_names']
+        file_name_template = file_names_per_platform.get(
+            bouncer_platform, file_names_per_platform.get('default', None)
+        )
+        if not file_name_template:
+            # Some bouncer product like stub-installer are only meant to be on Windows.
+            # Thus no default value is defined there
+            continue
+
+        file_name_product = _craft_filename_product(product)
+        file_name = file_name_template.format(
+            product=file_name_product,
+            pretty_product=file_name_product.capitalize(),
+            version=current_version,
+        )
+
+        path_template = CONFIG_PER_BOUNCER_PRODUCT[bouncer_product]['path_template']
+        file_relative_location = path_template.format(
+            ftp_product=_craft_ftp_product(product),
+            version=current_version,
+            ftp_platform=FTP_PLATFORMS_PER_BOUNCER_PLATFORM[bouncer_platform],
+            partner=partner,
+            sub_config=sub_config,
+            file=file_name,
+        )
+
+        paths_per_bouncer_platform[bouncer_platform] = file_relative_location
+
+    return paths_per_bouncer_platform
+
+
+def craft_partner_bouncer_product_name(product, bouncer_product, current_version,
+                                       partner, sub_config):
+    postfix = CONFIG_PER_BOUNCER_PRODUCT[bouncer_product].get('name_postfix', '').format(
+        partner=partner,
+        sub_config=sub_config,
+    )
+
+    return '{product}-{version}{postfix}'.format(
+        product=product.capitalize(), version=current_version, postfix=postfix
+    )
+
+
+def craft_ssl_only(bouncer_product):
+    return bouncer_product == "stub-installer"
--- a/taskcluster/taskgraph/util/partners.py
+++ b/taskcluster/taskgraph/util/partners.py
@@ -229,17 +229,18 @@ def get_partners(manifestRepo, token):
     return partners
 
 
 def parse_config(data):
     """ Parse a single repack.cfg file into a python dictionary.
     data is contents of the file, in "foo=bar\nbaz=buzz" style. We do some translation on
     locales and platforms data, otherwise passthrough
     """
-    ALLOWED_KEYS = ('locales', 'upload_to_candidates', 'repack_stub_installer', 'platforms')
+    ALLOWED_KEYS = ('locales', 'platforms', 'upload_to_candidates', 'repack_stub_installer',
+                    'publish_to_releases')
     config = {'platforms': []}
     for l in data.splitlines():
         if '=' in l:
             l = str(l)
             key, value = l.split('=', 1)
             value = value.strip('\'"').rstrip('\'"')
             if key in TC_PLATFORM_PER_FTP.keys():
                 if value.lower() == 'true':
@@ -398,8 +399,19 @@ def get_partner_url_config(parameters, g
         'release-level': release_level(parameters['project']),
         'release-type': parameters["release_type"]
     }
     resolve_keyed_by(partner_url_config, 'release-eme-free-repack', 'eme-free manifest_url',
                      **substitutions)
     resolve_keyed_by(partner_url_config, 'release-partner-repack', 'partner manifest url',
                      **substitutions)
     return partner_url_config
+
+
+def get_partners_to_be_published(config):
+    # hardcoded kind because release-bouncer-aliases doesn't match otherwise
+    partner_config = get_partner_config_by_kind(config, 'release-partner-repack')
+    partners = []
+    for partner, subconfigs in partner_config.items():
+        for sub_config_name, sub_config in subconfigs.items():
+            if sub_config.get("publish_to_releases"):
+                partners.append((partner, sub_config_name, sub_config['platforms']))
+    return partners