Bug 1630809 - generate beetmover tasks for partner attribution r=aki
authorNick Thomas <nthomas@mozilla.com>
Thu, 27 Aug 2020 22:19:34 +0000
changeset 546703 0015c6400e7242cdd90c9c14e99c668915b08c3b
parent 546702 e90a86c44d52b7470b8fa8df6022e1ad90bc907d
child 546704 09f2c990a88c4bb83ababa09f25a6ef0db4fae9a
push id37736
push userapavel@mozilla.com
push dateFri, 28 Aug 2020 15:31:26 +0000
treeherdermozilla-central@56166cae2e26 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki
bugs1630809
milestone82.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1630809 - generate beetmover tasks for partner attribution r=aki This is a fork of the release-partner-repack-beetmover kind and transform. It's modified to cope with having one upstream task with many partner builds, rather than a many beetmover tasks dealing with a single config-platform-locale combination. Differential Revision: https://phabricator.services.mozilla.com/D87730
taskcluster/ci/release-partner-attribution-beetmover/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/transforms/partner_attribution_beetmover.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/release-partner-attribution-beetmover/kind.yml
@@ -0,0 +1,23 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+    - taskgraph.transforms.name_sanity:transforms
+    - taskgraph.transforms.partner_attribution_beetmover:transforms
+    - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+    - release-partner-attribution
+
+job-template:
+    shipping-product: firefox
+    shipping-phase: promote
+    partner-bucket-scope:
+        by-release-level:
+            production: beetmover:bucket:partner
+            staging: beetmover:bucket:dep-partner
+    partner-public-path: "partner-repacks/{partner}/{subpartner}/v{release_partner_build_number}/{platform}/{locale}"
+    partner-private-path: "{partner}/{version}-{build_number}/{subpartner}/{platform}/{locale}"
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -482,16 +482,20 @@ Repackaging of partner repacks.
 release-partner-repack-repackage-signing
 ----------------------------------------
 External signing of partner repacks.
 
 release-partner-repack-beetmover
 --------------------------------
 Moves the partner repacks to S3 buckets.
 
+release-partner-attribution-beetmover
+-------------------------------------
+Moves the partner attributions to S3 buckets.
+
 release-partner-repack-bouncer-sub
 ----------------------------------
 Sets up bouncer products for partners.
 
 release-early-tagging
 ---------------------
 Utilises treescript to perform tagging that should happen near the start of a release.
 
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/partner_attribution_beetmover.py
@@ -0,0 +1,205 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from six import text_type
+from taskgraph.loader.single_dep import schema
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.beetmover import craft_release_properties
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.partners import (
+    get_partner_config_by_kind,
+    apply_partner_priority,
+)
+from taskgraph.util.schema import (
+    optionally_keyed_by,
+    resolve_keyed_by,
+)
+from taskgraph.util.scriptworker import (
+    add_scope_prefix,
+    get_beetmover_bucket_scope,
+)
+from taskgraph.util.taskcluster import get_artifact_prefix
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Any, Required, Optional
+
+from collections import defaultdict
+from copy import deepcopy
+
+
+beetmover_description_schema = schema.extend(
+    {
+        # depname is used in taskref's to identify the taskID of the unsigned things
+        Required("depname", default="build"): text_type,
+        # unique label to describe this beetmover task, defaults to {dep.label}-beetmover
+        Optional("label"): text_type,
+        Required("partner-bucket-scope"): optionally_keyed_by(
+            "release-level", text_type
+        ),
+        Required("partner-public-path"): Any(None, text_type),
+        Required("partner-private-path"): Any(None, text_type),
+        Optional("extra"): object,
+        Required("shipping-phase"): task_description_schema["shipping-phase"],
+        Optional("shipping-product"): task_description_schema["shipping-product"],
+        Optional("priority"): task_description_schema["priority"],
+    }
+)
+
+transforms = TransformSequence()
+transforms.add_validate(beetmover_description_schema)
+transforms.add(apply_partner_priority)
+
+
+@transforms.add
+def resolve_keys(config, jobs):
+    for job in jobs:
+        resolve_keyed_by(
+            job,
+            "partner-bucket-scope",
+            item_name=job["label"],
+            **{"release-level": config.params.release_level()}
+        )
+        yield job
+
+
+@transforms.add
+def split_public_and_private(config, jobs):
+    # we need to separate private vs public destinations because beetmover supports one
+    # in a single task. Only use a single task for each type though.
+    partner_config = get_partner_config_by_kind(config, config.kind)
+    for job in jobs:
+        upstream_artifacts = job["primary-dependency"].release_artifacts
+        attribution_task_ref = "<{}>".format(job["primary-dependency"].label)
+        prefix = get_artifact_prefix(job["primary-dependency"])
+        artifacts = defaultdict(list)
+        for artifact in upstream_artifacts:
+            partner, sub_partner, platform, locale, _ = artifact.replace(
+                prefix + "/", ""
+            ).split("/", 4)
+            destination = "private"
+            this_config = [p for p in partner_config["configs"] if (
+                    p["campaign"] == partner and p["content"] == sub_partner)
+            ]
+            if this_config[0].get("upload_to_candidates"):
+                destination = "public"
+            artifacts[destination].append(
+                (artifact, partner, sub_partner, platform, locale)
+            )
+
+        action_scope = add_scope_prefix(config, "beetmover:action:push-to-partner")
+        public_bucket_scope = get_beetmover_bucket_scope(config)
+        partner_bucket_scope = add_scope_prefix(config, job["partner-bucket-scope"])
+        repl_dict = {
+            "build_number": config.params["build_number"],
+            "release_partner_build_number": config.params[
+                "release_partner_build_number"
+            ],
+            "version": config.params["version"],
+            "partner": "{partner}",  # we'll replace these later, per artifact
+            "subpartner": "{subpartner}",
+            "platform": "{platform}",
+            "locale": "{locale}",
+        }
+        for destination, destination_artifacts in artifacts.items():
+            this_job = deepcopy(job)
+
+            if destination == "public":
+                this_job["scopes"] = [public_bucket_scope, action_scope]
+                this_job["partner_public"] = True
+            else:
+                this_job["scopes"] = [partner_bucket_scope, action_scope]
+                this_job["partner_public"] = False
+
+            partner_path_key = "partner-{destination}-path".format(
+                destination=destination
+            )
+            partner_path = this_job[partner_path_key].format(**repl_dict)
+            this_job.setdefault("worker", {})[
+                "upstream-artifacts"
+            ] = generate_upstream_artifacts(
+                attribution_task_ref, destination_artifacts, partner_path
+            )
+
+            yield this_job
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        dep_job = job["primary-dependency"]
+
+        attributes = dep_job.attributes
+        build_platform = attributes.get("build_platform")
+        if not build_platform:
+            raise Exception("Cannot find build platform!")
+
+        label = config.kind
+        description = "Beetmover for partner attribution"
+        if job["partner_public"]:
+            label = "{}-public".format(label)
+            description = "{} public".format(description)
+        else:
+            label = "{}-private".format(label)
+            description = "{} private".format(description)
+        attributes = copy_attributes_from_dependent_job(dep_job)
+
+        task = {
+            "label": label,
+            "description": description,
+            "dependencies": {dep_job.kind: dep_job.label},
+            "attributes": attributes,
+            "run-on-projects": dep_job.attributes.get("run_on_projects"),
+            "shipping-phase": job["shipping-phase"],
+            "shipping-product": job.get("shipping-product"),
+            "partner_public": job["partner_public"],
+            "worker": job["worker"],
+            "scopes": job["scopes"],
+        }
+        # we may have reduced the priority for partner jobs, otherwise task.py will set it
+        if job.get("priority"):
+            task["priority"] = job["priority"]
+
+        yield task
+
+
+def generate_upstream_artifacts(attribution_task, artifacts, partner_path):
+    upstream_artifacts = []
+    for artifact, partner, subpartner, platform, locale in artifacts:
+        upstream_artifacts.append(
+            {
+                "taskId": {"task-reference": attribution_task},
+                "taskType": "repackage",
+                "paths": [artifact],
+                "locale": partner_path.format(
+                    partner=partner,
+                    subpartner=subpartner,
+                    platform=platform,
+                    locale=locale,
+                ),
+            }
+        )
+
+    if not upstream_artifacts:
+        raise Exception("Couldn't find any upstream artifacts.")
+
+    return upstream_artifacts
+
+
+@transforms.add
+def make_task_worker(config, jobs):
+    for job in jobs:
+        job["worker-type"] = "beetmover"
+        worker = {
+            "implementation": "beetmover",
+            "release-properties": craft_release_properties(config, job),
+            "partner-public": job["partner_public"],
+        }
+        job["worker"].update(worker)
+        del job["partner_public"]
+
+        yield job