Bug 1433459 - part 1: Move bouncer submission tasks to scriptworker r=mtabara,rail
authorJohan Lorenzo <jlorenzo@mozilla.com>
Mon, 26 Feb 2018 15:14:46 +0100
changeset 461076 e91e79d2232ed4b0264a398e6aa8d341b38ddad6
parent 461075 a638660a49fa22780b515fb4770742d39c2105bf
child 461077 5396cec9cde697d4ad942c8526f9eb85042e294e
push id1683
push usersfraser@mozilla.com
push dateThu, 26 Apr 2018 16:43:40 +0000
treeherdermozilla-release@5af6cb21869d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmtabara, rail
bugs1433459
milestone60.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1433459 - part 1: Move bouncer submission tasks to scriptworker r=mtabara,rail MozReview-Commit-ID: 6SKhjf1ywoH
taskcluster/ci/release-bouncer-sub/kind.yml
taskcluster/taskgraph/transforms/bouncer_submission.py
taskcluster/taskgraph/transforms/l10n.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/release-bouncer-sub/kind.yml
+++ b/taskcluster/ci/release-bouncer-sub/kind.yml
@@ -1,41 +1,58 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 loader: taskgraph.loader.transform:loader
 
 transforms:
-   - taskgraph.transforms.job:transforms
+   - taskgraph.transforms.bouncer_submission:transforms
    - taskgraph.transforms.release_notifications:transforms
    - taskgraph.transforms.task:transforms
 
 job-defaults:
    description: release bouncer submission job
-   worker-type: buildbot-bridge/buildbot-bridge
+   worker-type:
+      by-project:
+         mozilla-central: scriptworker-prov-v1/bouncer-v1
+         mozilla-beta: scriptworker-prov-v1/bouncer-v1
+         mozilla-release: scriptworker-prov-v1/bouncer-v1
+         default: scriptworker-prov-v1/bouncer-dev
+   worker:
+      implementation: bouncer-submission
+   scopes:
+      by-project:
+         mozilla-beta:
+            - project:releng:bouncer:action:submission
+            - project:releng:bouncer:server:production
+         mozilla-release:
+            - project:releng:bouncer:action:submission
+            - project:releng:bouncer:server:production
+         default:
+            - project:releng:bouncer:action:submission
+            - project:releng:bouncer:server:staging
    run-on-projects: []
    shipping-phase: promote
-   run:
-      using: buildbot
-      release-promotion: true
+   shipping-product: firefox
+   locales-file: browser/locales/l10n-changesets.json
 
 jobs:
+   devedition:
+      bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64']
+      bouncer-products: ['complete-mar', 'installer', 'installer-ssl', 'partial-mar', 'stub-installer']
+      shipping-product: devedition
+
    fennec:
-      name: fennec_release_bouncer_sub
+      bouncer-platforms: ['android', 'android-x86']
+      bouncer-products: ['apk']
       shipping-product: fennec
-      run:
-         product: fennec
-         buildername: release-{branch}-fennec_bncr_sub
+      locales-file: mobile/locales/l10n-changesets.json
 
    firefox:
-      name: firefox_release_bouncer_sub
+      bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64']
+      bouncer-products: ['complete-mar', 'installer', 'installer-ssl', 'partial-mar', 'stub-installer']
       shipping-product: firefox
-      run:
-         product: firefox
-         buildername: release-{branch}_firefox_bncr_sub
 
-   devedition:
-      name: devedition_release_bouncer_sub
-      shipping-product: devedition
-      run:
-         product: devedition
-         buildername: release-{branch}_devedition_bncr_sub
+   firefox-rc:
+      bouncer-platforms: ['linux', 'linux64', 'osx', 'win', 'win64']
+      bouncer-products: ['partial-mar-candidates']
+      shipping-product: firefox
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/bouncer_submission.py
@@ -0,0 +1,253 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Add from parameters.yml into bouncer submission tasks.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.l10n import parse_locales_file
+from taskgraph.util.schema import resolve_keyed_by
+from taskgraph.util.scriptworker import get_release_config
+
+logger = logging.getLogger(__name__)
+
+
+FTP_PLATFORMS_PER_BOUNCER_PLATFORM = {
+    'android': 'android-api-16',
+    'android-x86': 'android-x86',
+    'linux': 'linux-i686',
+    'linux64': 'linux-x86_64',
+    'osx': 'mac',
+    'win': 'win32',
+    'win64': 'win64',
+}
+
+# :lang is interpolated by bouncer at runtime
+CANDIDATES_PATH_TEMPLATE = '/{product}/candidates/{version}-candidates/build{build_number}/\
+{update_folder}{ftp_platform}/:lang/{file}'
+RELEASES_PATH_TEMPLATE = '/{product}/releases/{version}/{update_folder}{ftp_platform}/:lang/{file}'
+
+
+CONFIG_PER_BOUNCER_PRODUCT = {
+    'apk': {
+        'path_template': RELEASES_PATH_TEMPLATE,
+        'file_names': {
+            'android': 'fennec-{version}.:lang.android-arm.apk',
+            'android-x86': 'fennec-{version}.:lang.android-i386.apk',
+        },
+    },
+    'complete-mar': {
+        'path_template': RELEASES_PATH_TEMPLATE,
+        'file_names': {
+            'default': 'firefox-{version}.complete.mar',
+        },
+    },
+    'installer': {
+        'path_template': RELEASES_PATH_TEMPLATE,
+        'file_names': {
+            'linux': 'firefox-{version}.tar.bz2',
+            'linux64': 'firefox-{version}.tar.bz2',
+            'osx': 'Firefox%20{version}.dmg',
+            'win': 'Firefox%20Setup%20{version}.exe',
+            'win64': 'Firefox%20Setup%20{version}.exe',
+        },
+    },
+    'partial-mar': {
+        'path_template': RELEASES_PATH_TEMPLATE,
+        'file_names': {
+            'default': 'firefox-{previous_version}-{version}.partial.mar',
+        },
+    },
+    'partial-mar-candidates': {
+        'path_template': CANDIDATES_PATH_TEMPLATE,
+        'file_names': {
+            'default': 'firefox-{previous_version}-{version}.partial.mar',
+        },
+    },
+    'stub-installer': {
+        'path_template': RELEASES_PATH_TEMPLATE,
+        'file_names': {
+            'win': 'Firefox%20Installer.exe',
+            'win64': 'Firefox%20Installer.exe',
+        },
+    },
+}
+CONFIG_PER_BOUNCER_PRODUCT['installer-ssl'] = CONFIG_PER_BOUNCER_PRODUCT['installer']
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def make_task_worker(config, jobs):
+    for job in jobs:
+        resolve_keyed_by(
+            job, 'worker-type', item_name=job['name'], project=config.params['project']
+        )
+        resolve_keyed_by(
+            job, 'scopes', item_name=job['name'], project=config.params['project']
+        )
+
+        # No need to filter out ja-JP-mac, we need to upload both
+        all_locales = list(sorted(parse_locales_file(job['locales-file']).keys()))
+        job['worker']['locales'] = all_locales
+        job['worker']['entries'] = craft_bouncer_entries(config, job)
+
+        del job['locales-file']
+        del job['bouncer-platforms']
+        del job['bouncer-products']
+
+        if job['worker']['entries']:
+            # XXX Because rc jobs are defined within the same kind, we need to delete the
+            # firefox-rc job at this stage, if we're not building an RC. Otherwise, even if
+            # target_tasks.py filters out the rc job, it gets resurected by any kind that depends
+            # on the release-bouncer-sub one (release-notify-promote as of time of this writing).
+            if config.params['release_type'] == 'rc' or job['name'] != 'firefox-rc':
+                yield job
+        else:
+            logger.warn('No bouncer entries defined in bouncer submission task for "{}". \
+Job deleted.'.format(job['name']))
+
+
+def craft_bouncer_entries(config, job):
+    release_config = get_release_config(config)
+
+    product = job['shipping-product']
+    bouncer_platforms = job['bouncer-platforms']
+
+    current_version = release_config['version']
+    current_build_number = release_config['build_number']
+
+    bouncer_products = job['bouncer-products']
+    previous_versions_string = release_config.get('partial_versions', None)
+    if previous_versions_string:
+        previous_versions = previous_versions_string.split(', ')
+    else:
+        logger.warn('No partials defined! Bouncer submission task won\'t send any \
+partial-related entry for "{}"'.format(job['name']))
+        bouncer_products = [
+            bouncer_product
+            for bouncer_product in bouncer_products
+            if 'partial' not in bouncer_product
+        ]
+        previous_versions = [None]
+
+    project = config.params['project']
+
+    return {
+        craft_bouncer_product_name(
+            product, bouncer_product, current_version, current_build_number, previous_version
+        ): {
+            'options': {
+                'add_locales': craft_add_locales(product),
+                'check_uptake': craft_check_uptake(bouncer_product),
+                'ssl_only': craft_ssl_only(bouncer_product, project),
+            },
+            'paths_per_bouncer_platform': craft_paths_per_bouncer_platform(
+                product, bouncer_product, bouncer_platforms, current_version,
+                current_build_number, previous_version
+            ),
+        }
+        for bouncer_product in bouncer_products
+        for previous_version in previous_versions
+    }
+
+
+def craft_paths_per_bouncer_platform(product, bouncer_product, bouncer_platforms, current_version,
+                                     current_build_number, previous_version=None):
+    paths_per_bouncer_platform = {}
+    for bouncer_platform in bouncer_platforms:
+        ftp_platform = FTP_PLATFORMS_PER_BOUNCER_PLATFORM[bouncer_platform]
+
+        file_names_per_platform = CONFIG_PER_BOUNCER_PRODUCT[bouncer_product]['file_names']
+        file_name_template = file_names_per_platform.get(
+            bouncer_platform, file_names_per_platform.get('default', None)
+        )
+        if not file_name_template:
+            # Some bouncer product like stub-installer are only meant to be on Windows.
+            # Thus no default value is defined there
+            continue
+
+        file_name = file_name_template.format(
+            version=current_version, previous_version=strip_build_data(previous_version)
+        )
+
+        path_template = CONFIG_PER_BOUNCER_PRODUCT[bouncer_product]['path_template']
+        file_relative_location = path_template.format(
+            product=product.lower(),
+            version=current_version,
+            build_number=current_build_number,
+            update_folder='updates/' if '-mar' in bouncer_product else '',
+            ftp_platform=ftp_platform,
+            file=file_name,
+        )
+
+        paths_per_bouncer_platform[bouncer_platform] = file_relative_location
+
+    return paths_per_bouncer_platform
+
+
+def craft_bouncer_product_name(product, bouncer_product, current_version,
+                               current_build_number=None, previous_version=None):
+    if '-ssl' in bouncer_product:
+        postfix = '-SSL'
+    elif 'stub-' in bouncer_product:
+        postfix = '-stub'
+    elif 'complete-' in bouncer_product:
+        postfix = '-Complete'
+    elif 'partial-' in bouncer_product:
+        if not previous_version:
+            raise Exception('Partial is being processed, but no previous version defined.')
+
+        if '-candidates' in bouncer_product:
+            if not current_build_number:
+                raise Exception('Partial in candidates directory is being processed, \
+but no current build number defined.')
+
+            postfix = 'build{build_number}-Partial-{previous_version_with_build_number}'.format(
+                build_number=current_build_number,
+                previous_version_with_build_number=previous_version,
+            )
+        else:
+            postfix = '-Partial-{previous_version}'.format(
+                previous_version=strip_build_data(previous_version)
+            )
+
+    elif 'sha1-' in bouncer_product:
+        postfix = '-sha1'
+    else:
+        postfix = ''
+
+    return '{product}-{version}{postfix}'.format(
+        product=product.capitalize(), version=current_version, postfix=postfix
+    )
+
+
+def craft_check_uptake(bouncer_product):
+    return bouncer_product != 'complete-mar-candidates'
+
+
+def craft_ssl_only(bouncer_product, project):
+    # XXX ESR is the only channel where we force serve the installer over SSL
+    if '-esr' in project and bouncer_product == 'installer':
+        return True
+
+    return bouncer_product not in (
+        'complete-mar',
+        'installer',
+        'partial-mar',
+        'partial-mar-candidates',
+    )
+
+
+def craft_add_locales(product):
+    # Do not add locales on Fennec in order to let "multi" work
+    return product != 'fennec'
+
+
+def strip_build_data(version):
+    return version.split('build')[0] if version and 'build' in version else version
--- a/taskcluster/taskgraph/transforms/l10n.py
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -158,29 +158,29 @@ l10n_description_schema = Schema({
     # Shipping product and phase
     Optional('shipping-product'): task_description_schema['shipping-product'],
     Optional('shipping-phase'): task_description_schema['shipping-phase'],
 })
 
 transforms = TransformSequence()
 
 
-def _parse_locales_file(locales_file, platform):
+def parse_locales_file(locales_file, platform=None):
     """ Parse the passed locales file for a list of locales.
     """
     locales = []
 
     with open(locales_file, mode='r') as f:
         if locales_file.endswith('json'):
             all_locales = json.load(f)
             # XXX Only single locales are fetched
             locales = {
                 locale: data['revision']
                 for locale, data in all_locales.items()
-                if platform in data['platforms']
+                if platform is None or platform in data['platforms']
             }
         else:
             all_locales = f.read().split()
             # 'default' is the hg revision at the top of hg repo, in this context
             locales = {locale: 'default' for locale in all_locales}
     return locales
 
 
@@ -294,18 +294,18 @@ def handle_keyed_by(config, jobs):
             resolve_keyed_by(item=job, field=field, item_name=job['name'])
         yield job
 
 
 @transforms.add
 def all_locales_attribute(config, jobs):
     for job in jobs:
         locales_platform = job['attributes']['build_platform'].replace("-nightly", "")
-        locales_with_changesets = _parse_locales_file(job["locales-file"],
-                                                      platform=locales_platform)
+        locales_with_changesets = parse_locales_file(job["locales-file"],
+                                                     platform=locales_platform)
         locales_with_changesets = _remove_locales(locales_with_changesets,
                                                   to_remove=job['ignore-locales'])
 
         locales = sorted(locales_with_changesets.keys())
         attributes = job.setdefault('attributes', {})
         attributes["all_locales"] = locales
         attributes["all_locales_with_changesets"] = locales_with_changesets
         if job.get('shipping-product'):
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -554,19 +554,22 @@ task_description_schema = Schema({
 
             # type of signing task (for CoT)
             Required('taskType'): basestring,
 
             # Paths to the artifacts to sign
             Required('paths'): [basestring],
         }],
     }, {
+        Required('implementation'): 'bouncer-submission',
+        Required('locales'): [basestring],
+        Required('entries'): object,
+    }, {
         Required('implementation'): 'push-apk-breakpoint',
         Required('payload'): object,
-
     }, {
         Required('implementation'): 'invalid',
         # an invalid task is one which should never actually be created; this is used in
         # release automation on branches where the task just doesn't make sense
         Extra: object,
 
     }, {
         Required('implementation'): 'always-optimized',
@@ -1107,16 +1110,26 @@ def build_balrog_payload(config, task, t
             })
         else:  # schedule / ship
             task_def['payload'].update({
                 'publish_rules': worker['publish-rules'],
                 'release_eta': config.params.get('release_eta') or '',
             })
 
 
+@payload_builder('bouncer-submission')
+def build_bouncer_submission_payload(config, task, task_def):
+    worker = task['worker']
+
+    task_def['payload'] = {
+        'locales':  worker['locales'],
+        'submission_entries': worker['entries']
+    }
+
+
 @payload_builder('push-apk')
 def build_push_apk_payload(config, task, task_def):
     worker = task['worker']
 
     task_def['payload'] = {
         'commit': worker['commit'],
         'upstreamArtifacts':  worker['upstream-artifacts'],
         'google_play_track': worker['google-play-track'],