taskcluster/taskgraph/transforms/beetmover_repackage_partner.py
author Alex Chronopoulos <achronop@gmail.com>
Thu, 02 Jul 2020 12:20:24 +0000
changeset 538410 2d709e60c76ec75960054105fae65a758d7a5e78
parent 511495 c933f5b712cda884912440d7577e6af85ef6b468
permissions -rw-r--r--
Bug 1649058 - Replace MozPromise with SyncRunnable and assert not main thread. r=jya Differential Revision: https://phabricator.services.mozilla.com/D81998

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Transform the beetmover task into an actual task description.
"""

from __future__ import absolute_import, print_function, unicode_literals

from six import text_type
from taskgraph.loader.single_dep import schema
from taskgraph.transforms.base import TransformSequence
from taskgraph.transforms.beetmover import craft_release_properties
from taskgraph.util.attributes import copy_attributes_from_dependent_job
from taskgraph.util.partners import (
    check_if_partners_enabled,
    get_ftp_platform,
    get_partner_config_by_kind,
)
from taskgraph.util.schema import (
    optionally_keyed_by,
    resolve_keyed_by,
)
from taskgraph.util.scriptworker import (
    add_scope_prefix,
    get_beetmover_bucket_scope,
)
from taskgraph.util.taskcluster import get_artifact_prefix
from taskgraph.transforms.task import task_description_schema
from voluptuous import Any, Required, Optional

from copy import deepcopy
import logging

logger = logging.getLogger(__name__)


beetmover_description_schema = schema.extend({
    # depname is used in taskref's to identify the taskID of the unsigned things
    Required('depname', default='build'): text_type,

    # unique label to describe this beetmover task, defaults to {dep.label}-beetmover
    Optional('label'): text_type,

    Required('partner-bucket-scope'): optionally_keyed_by('release-level', text_type),
    Required('partner-public-path'): Any(None, text_type),
    Required('partner-private-path'): Any(None, text_type),

    Optional('extra'): object,
    Required('shipping-phase'): task_description_schema['shipping-phase'],
    Optional('shipping-product'): task_description_schema['shipping-product'],
    Optional('priority'): task_description_schema['priority'],
})

transforms = TransformSequence()
transforms.add(check_if_partners_enabled)
transforms.add_validate(beetmover_description_schema)


@transforms.add
def resolve_keys(config, jobs):
    for job in jobs:
        resolve_keyed_by(
            job, 'partner-bucket-scope', item_name=job['label'],
            **{'release-level': config.params.release_level()}
        )
        yield job


@transforms.add
def make_task_description(config, jobs):
    for job in jobs:
        dep_job = job['primary-dependency']
        repack_id = dep_job.task.get('extra', {}).get('repack_id')
        if not repack_id:
            raise Exception("Cannot find repack id!")

        attributes = dep_job.attributes
        build_platform = attributes.get("build_platform")
        if not build_platform:
            raise Exception("Cannot find build platform!")

        label = dep_job.label.replace("repackage-signing-l10n", "beetmover-")
        label = dep_job.label.replace("repackage-signing-", "beetmover-")
        label = label.replace("repackage-", "beetmover-")
        label = label.replace("chunking-dummy-", "beetmover-")
        description = (
            "Beetmover submission for repack_id '{repack_id}' for build '"
            "{build_platform}/{build_type}'".format(
                repack_id=repack_id,
                build_platform=build_platform,
                build_type=attributes.get('build_type')
            )
        )

        dependencies = {}

        base_label = "release-partner-repack"
        if "eme" in config.kind:
            base_label = "release-eme-free-repack"
        dependencies["build"] = "{}-{}".format(base_label, build_platform)
        if "macosx" in build_platform or "win" in build_platform:
            dependencies["repackage"] = "{}-repackage-{}-{}".format(
                base_label, build_platform, repack_id.replace('/', '-')
            )
        dependencies["repackage-signing"] = "{}-repackage-signing-{}-{}".format(
             base_label, build_platform, repack_id.replace('/', '-')
        )

        attributes = copy_attributes_from_dependent_job(dep_job)

        task = {
            'label': label,
            'description': description,
            'dependencies': dependencies,
            'attributes': attributes,
            'run-on-projects': dep_job.attributes.get('run_on_projects'),
            'shipping-phase': job['shipping-phase'],
            'shipping-product': job.get('shipping-product'),
            'partner-private-path': job['partner-private-path'],
            'partner-public-path': job['partner-public-path'],
            'partner-bucket-scope': job['partner-bucket-scope'],
            'extra': {
                'repack_id': repack_id,
            },
        }
        # we may have reduced the priority for partner jobs, otherwise task.py will set it
        if job.get('priority'):
            task['priority'] = job['priority']

        yield task


def populate_scopes_and_worker_type(config, job, bucket_scope, partner_public=False):
    action_scope = add_scope_prefix(config, 'beetmover:action:push-to-partner')

    task = deepcopy(job)
    task['scopes'] = [bucket_scope, action_scope]
    task['worker-type'] = 'beetmover'
    task['partner_public'] = partner_public
    if partner_public:
        task['label'] = "{}-public".format(task['label'])
    return task


@transforms.add
def split_public_and_private(config, jobs):
    public_bucket_scope = get_beetmover_bucket_scope(config)
    partner_config = get_partner_config_by_kind(config, config.kind)

    for job in jobs:
        partner_bucket_scope = add_scope_prefix(config, job['partner-bucket-scope'])
        partner, subpartner, _ = job['extra']['repack_id'].split('/')

        if partner_config[partner][subpartner].get('upload_to_candidates'):
            # public
            yield populate_scopes_and_worker_type(
                config, job, public_bucket_scope, partner_public=True
            )
        else:
            # private
            yield populate_scopes_and_worker_type(
                config, job, partner_bucket_scope, partner_public=False
            )


def generate_upstream_artifacts(job, build_task_ref, repackage_task_ref,
                                repackage_signing_task_ref, platform, repack_id,
                                partner_path, repack_stub_installer=False):

    upstream_artifacts = []
    artifact_prefix = get_artifact_prefix(job)

    if "linux" in platform:
        upstream_artifacts.append({
            "taskId": {"task-reference": build_task_ref},
            "taskType": "build",
            "paths": ["{}/{}/target.tar.bz2".format(artifact_prefix, repack_id)],
            "locale": partner_path,
        })
        upstream_artifacts.append({
            "taskId": {"task-reference": repackage_signing_task_ref},
            "taskType": "repackage",
            "paths": ["{}/{}/target.tar.bz2.asc".format(artifact_prefix, repack_id)],
            "locale": partner_path,
        })
    elif "macosx" in platform:
        upstream_artifacts.append({
            "taskId": {"task-reference": repackage_task_ref},
            "taskType": "repackage",
            "paths": ["{}/{}/target.dmg".format(artifact_prefix, repack_id)],
            "locale": partner_path,
        })
        upstream_artifacts.append({
            "taskId": {"task-reference": repackage_signing_task_ref},
            "taskType": "repackage",
            "paths": ["{}/{}/target.dmg.asc".format(artifact_prefix, repack_id)],
            "locale": partner_path,
        })
    elif "win" in platform:
        upstream_artifacts.append({
            "taskId": {"task-reference": repackage_signing_task_ref},
            "taskType": "repackage",
            "paths": ["{}/{}/target.installer.exe".format(artifact_prefix, repack_id)],
            "locale": partner_path,
        })
        upstream_artifacts.append({
            "taskId": {"task-reference": repackage_signing_task_ref},
            "taskType": "repackage",
            "paths": ["{}/{}/target.installer.exe.asc".format(artifact_prefix, repack_id)],
            "locale": partner_path,
        })
        if platform.startswith('win32') and repack_stub_installer:
            upstream_artifacts.append({
                "taskId": {"task-reference": repackage_signing_task_ref},
                "taskType": "repackage",
                "paths": ["{}/{}/target.stub-installer.exe".format(artifact_prefix, repack_id)],
                "locale": partner_path,
            })
            upstream_artifacts.append({
                "taskId": {"task-reference": repackage_signing_task_ref},
                "taskType": "repackage",
                "paths": ["{}/{}/target.stub-installer.exe.asc".format(
                    artifact_prefix, repack_id)],
                "locale": partner_path,
            })

    if not upstream_artifacts:
        raise Exception("Couldn't find any upstream artifacts.")

    return upstream_artifacts


@transforms.add
def make_task_worker(config, jobs):
    for job in jobs:
        platform = job["attributes"]["build_platform"]
        repack_id = job["extra"]["repack_id"]
        partner, subpartner, locale = job['extra']['repack_id'].split('/')
        partner_config = get_partner_config_by_kind(config, config.kind)
        repack_stub_installer = partner_config[partner][subpartner].get('repack_stub_installer')
        build_task = None
        repackage_task = None
        repackage_signing_task = None

        for dependency in job["dependencies"].keys():
            if 'repackage-signing' in dependency:
                repackage_signing_task = dependency
            elif 'repackage' in dependency:
                repackage_task = dependency
            else:
                build_task = "build"

        build_task_ref = "<" + str(build_task) + ">"
        repackage_task_ref = "<" + str(repackage_task) + ">"
        repackage_signing_task_ref = "<" + str(repackage_signing_task) + ">"

        # generate the partner path; we'll send this to beetmover as the "locale"
        ftp_platform = get_ftp_platform(platform)
        repl_dict = {
            "build_number": config.params['build_number'],
            "locale": locale,
            "partner": partner,
            "platform": ftp_platform,
            "release_partner_build_number": config.params['release_partner_build_number'],
            "subpartner": subpartner,
            "version": config.params['version'],
        }
        partner_public = job['partner_public']
        if partner_public:
            partner_path_key = 'partner-public-path'
        else:
            partner_path_key = 'partner-private-path'
        # Kinds can set these to None
        if not job[partner_path_key]:
            continue
        partner_path = job[partner_path_key].format(**repl_dict)
        del(job['partner_public'])
        del(job['partner-private-path'])
        del(job['partner-public-path'])
        del(job['partner-bucket-scope'])

        worker = {
            'implementation': 'beetmover',
            'release-properties': craft_release_properties(config, job),
            'upstream-artifacts': generate_upstream_artifacts(
                job, build_task_ref, repackage_task_ref,
                repackage_signing_task_ref, platform, repack_id,
                partner_path, repack_stub_installer
            ),
            'partner-public': partner_public,
        }
        job["worker"] = worker

        yield job