Bug 1330668 - Schedule beetmover tasks. r=dustin
authorJustin Wood <Callek@gmail.com>
Thu, 12 Jan 2017 17:45:36 -0500
changeset 462765 028c582db1762776cfa03f91805f74ba5e4453a5
parent 462764 023c679970910012d191a131218414220ed19121
child 462766 96ccea5f9ed9902a7ba4521bc84d136e791017f3
push id41857
push userbmo:mh+mozilla@glandium.org
push dateWed, 18 Jan 2017 00:24:11 +0000
reviewersdustin
bugs1330668
milestone53.0a1
Bug 1330668 - Schedule beetmover tasks. r=dustin ToDo: attribute this cset to changes on date. MozReview-Commit-ID: 7gl3aLGNWvP
taskcluster/ci/beetmover-l10n/kind.yml
taskcluster/ci/beetmover/kind.yml
taskcluster/docs/attributes.rst
taskcluster/docs/kinds.rst
taskcluster/taskgraph/task/beetmover.py
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/beetmover_l10n.py
taskcluster/taskgraph/transforms/task.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/beetmover-l10n/kind.yml
@@ -0,0 +1,14 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.beetmover:BeetmoverTask
+
+transforms:
+   - taskgraph.transforms.beetmover_l10n:transforms
+   - taskgraph.transforms.beetmover:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - nightly-l10n
+  - nightly-l10n-signing
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/beetmover/kind.yml
@@ -0,0 +1,13 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.beetmover:BeetmoverTask
+
+transforms:
+   - taskgraph.transforms.beetmover:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - build
+  - build-signing
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -133,8 +133,14 @@ l10n_chunk
 ==========
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the chunk
 number of the job. Note that this is a string!
 
 chunk_locales
 =============
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains an array of
 the individual locales this chunk is responsible for processing.
+
+locale
+======
+For jobs that operate on only one locale, we set the attribute ``locale`` to the
+specific locale involved. Currently this is only in l10n versions of the
+``beetmover`` kinds.
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -154,8 +154,24 @@ depend on another in-tree docker image, 
 Docker repository
 
 The task definition used to create the image-building tasks is given in
 ``image.yml`` in the kind directory, and is interpreted as a :doc:`YAML
 Template <yaml-templates>`.
 
 android-stuff
 --------------
+
+beetmover
+---------
+
+Beetmover, takes specific artifacts, "Beets", and pushes them to a location outside
+of Taskcluster's task artifacts, (archive.mozilla.org as one place) and in the
+process determines the final location and a "pretty" name (versioned product name)
+
+beetmover-l10n
+--------------
+
+Beetmover L10n, takes specific artifacts, "Beets", and pushes them to a location outside
+of Taskcluster's task artifacts, (archive.mozilla.org as one place) and in the
+process determines the final location and a "pretty" name (versioned product name)
+This separate kind uses logic specific to localized artifacts, such as including
+the language in the final artifact names.
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/task/beetmover.py
@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from . import transform
+
+
+class BeetmoverTask(transform.TransformTask):
+    """
+    A task implementing a signing job.  These depend on nightly build jobs and
+    sign the artifacts after a build has completed.
+
+    We use a dictionary to create the input to the transforms.
+    It will have added to it keys `build-label`, the label for the build task,
+    and `build-platform` / `build-type`, its platform and type.
+    """
+
+    @classmethod
+    def get_inputs(cls, kind, path, config, params, loaded_tasks):
+        if config.get('kind-dependencies', []) != ["build", "build-signing"] and \
+           config.get('kind-dependencies', []) != ["nightly-l10n", "nightly-l10n-signing"]:
+            raise Exception("Beetmover kinds must depend on builds or signing builds")
+        for task in loaded_tasks:
+            if not task.attributes.get('nightly'):
+                continue
+            if task.kind not in config.get('kind-dependencies'):
+                continue
+            beetmover_task = {}
+            beetmover_task['dependent-task'] = task
+
+            yield beetmover_task
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -0,0 +1,330 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import (
+    validate_schema,
+    TransformSequence
+)
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Schema, Any, Required, Optional
+
+
+_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
+    "balrog_props.json",
+    "target.common.tests.zip",
+    "target.cppunittest.tests.zip",
+    "target.crashreporter-symbols.zip",
+    "target.json",
+    "target.mochitest.tests.zip",
+    "target.mozinfo.json",
+    "target.reftest.tests.zip",
+    "target.talos.tests.zip",
+    "target.test_packages.json",
+    "target.txt",
+    "target.web-platform.tests.zip",
+    "target.xpcshell.tests.zip",
+    "target_info.txt",
+    "target.jsshell.zip",
+    "mozharness.zip",
+    "target.langpack.xpi",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US = [
+    "update/target.complete.mar",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N = [
+    "target.langpack.xpi",
+    "balrog_props.json",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N = [
+    "target.complete.mar",
+]
+_MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
+    "en-US/target.common.tests.zip",
+    "en-US/target.cppunittest.tests.zip",
+    "en-US/target.json",
+    "en-US/target.mochitest.tests.zip",
+    "en-US/target.mozinfo.json",
+    "en-US/target.reftest.tests.zip",
+    "en-US/target.talos.tests.zip",
+    "en-US/target.test_packages.json",
+    "en-US/target.txt",
+    "en-US/target.web-platform.tests.zip",
+    "en-US/target.xpcshell.tests.zip",
+    "en-US/target_info.txt",
+    "en-US/bouncer.apk",
+    "en-US/mozharness.zip",
+    "en-US/robocop.apk",
+    "en-US/target.jsshell.zip",
+]
+_MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI = [
+    "balrog_props.json",
+    "target.common.tests.zip",
+    "target.cppunittest.tests.zip",
+    "target.json",
+    "target.mochitest.tests.zip",
+    "target.mozinfo.json",
+    "target.reftest.tests.zip",
+    "target.talos.tests.zip",
+    "target.test_packages.json",
+    "target.txt",
+    "target.web-platform.tests.zip",
+    "target.xpcshell.tests.zip",
+    "target_info.txt",
+    "bouncer.apk",
+    "mozharness.zip",
+    "robocop.apk",
+    "target.jsshell.zip",
+]
+_MOBILE_UPSTREAM_ARTIFACTS_SIGNED_EN_US = [
+    "en-US/target.apk",
+]
+_MOBILE_UPSTREAM_ARTIFACTS_SIGNED_MULTI = [
+    "target.apk",
+]
+
+
+UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
+    'linux64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
+        "target.sdk.tar.bz2"
+    ],
+    'linux-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
+        "target.sdk.tar.bz2"
+    ],
+    'android-x86-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
+    'android-api-15-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
+    'macosx64-nightly': [],
+
+    'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
+    'linux-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
+    'android-x86-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
+    'android-api-15-nightly-l10n': ["balrog_props.json"],
+    'android-api-15-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
+    'macosx64-nightly-l10n': [],
+}
+UPSTREAM_ARTIFACT_SIGNED_PATHS = {
+    'linux64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
+        "target.tar.bz2",
+        "target.tar.bz2.asc",
+    ],
+    'linux-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
+        "target.tar.bz2",
+        "target.tar.bz2.asc",
+    ],
+    'android-x86-nightly': ["en-US/target.apk"],
+    'android-api-15-nightly': ["en-US/target.apk"],
+    'macosx64-nightly': [],
+
+    'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N + [
+        "target.tar.bz2",
+        "target.tar.bz2.asc",
+    ],
+    'linux-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N + [
+        "target.tar.bz2",
+        "target.tar.bz2.asc",
+    ],
+    'android-x86-nightly-multi': ["target.apk"],
+    'android-api-15-nightly-l10n': ["target.apk"],
+    'android-api-15-nightly-multi': ["target.apk"],
+    'macosx64-nightly-l10n': [],
+}
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+transforms = TransformSequence()
+
+# shortcut for a string where task references are allowed
+taskref_or_string = Any(
+    basestring,
+    {Required('task-reference'): basestring})
+
+beetmover_description_schema = Schema({
+    # the dependent task (object) for this beetmover job, used to inform beetmover.
+    Required('dependent-task'): object,
+
+    # depname is used in taskref's to identify the taskID of the unsigned things
+    Required('depname', default='build'): basestring,
+
+    # unique label to describe this beetmover task, defaults to {dep.label}-beetmover
+    Optional('label'): basestring,
+
+    # treeherder is allowed here to override any defaults we use for beetmover.  See
+    # taskcluster/taskgraph/transforms/task.py for the schema details, and the
+    # below transforms for defaults of various values.
+    Optional('treeherder'): task_description_schema['treeherder'],
+
+    # locale is passed only for l10n beetmoving
+    Optional('locale'): basestring,
+})
+
+
+@transforms.add
+def validate(config, jobs):
+    for job in jobs:
+        label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+        yield validate_schema(
+            beetmover_description_schema, job,
+            "In beetmover ({!r} kind) task for {!r}:".format(config.kind, label))
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+
+        treeherder = job.get('treeherder', {})
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform',
+                              "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('tier', 2)
+        treeherder.setdefault('kind', 'build')
+
+        label = job.get('label', "beetmover-{}".format(dep_job.label))
+        dependent_kind = str(dep_job.kind)
+        dependencies = {dependent_kind: dep_job.label}
+        # taskid_of_manifest always refers to the unsigned task
+        if "signing" in dependent_kind:
+            if len(dep_job.dependencies) > 1:
+                raise NotImplementedError(
+                    "can't beetmove a signing task with multiple dependencies")
+            signing_dependencies = dep_job.dependencies
+            dependencies.update(signing_dependencies)
+            treeherder.setdefault('symbol', 'tc(BM-S)')
+        else:
+            treeherder.setdefault('symbol', 'tc(BM)')
+
+        attributes = {
+                'nightly': dep_job.attributes.get('nightly', False),
+                'build_platform': dep_job.attributes.get('build_platform'),
+                'build_type': dep_job.attributes.get('build_type'),
+        }
+        if job.get('locale'):
+            attributes['locale'] = job['locale']
+
+        task = {
+            'label': label,
+            'description': "{} Beetmover".format(
+                dep_job.task["metadata"]["description"]),
+            # do we have to define worker type somewhere?
+            'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
+            # bump this to nightly / release when applicable+permitted
+            'scopes': ["project:releng:beetmover:nightly"],
+            'dependencies': dependencies,
+            'attributes': attributes,
+            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'treeherder': treeherder,
+        }
+
+        yield task
+
+
+def generate_upstream_artifacts(taskid_to_beetmove, platform, locale=None, signing=False):
+    task_type = "build"
+    mapping = UPSTREAM_ARTIFACT_UNSIGNED_PATHS
+    if signing:
+        task_type = "signing"
+        mapping = UPSTREAM_ARTIFACT_SIGNED_PATHS
+
+    artifact_prefix = 'public/build'
+    if locale:
+        artifact_prefix = 'public/build/{}'.format(locale)
+        platform = "{}-l10n".format(platform)
+
+    upstream_artifacts = [{
+        "taskId": {"task-reference": taskid_to_beetmove},
+        "taskType": task_type,
+        "paths": ["{}/{}".format(artifact_prefix, p) for p in mapping[platform]],
+        "locale": locale or "en-US",
+    }]
+    if not locale and "android" in platform:
+        # edge case to support 'multi' locale paths
+        multi_platform = "{}-multi".format(platform)
+        upstream_artifacts.append({
+            "taskId": {"task-reference": taskid_to_beetmove},
+            "taskType": task_type,
+            "paths": ["{}/{}".format(artifact_prefix, p) for p in mapping[multi_platform]],
+            "locale": "multi",
+        })
+
+    return upstream_artifacts
+
+
+def generate_signing_upstream_artifacts(taskid_to_beetmove, taskid_of_manifest, platform,
+                                        locale=None):
+    upstream_artifacts = generate_upstream_artifacts(taskid_to_beetmove, platform, locale,
+                                                     signing=True)
+    if locale:
+        artifact_prefix = 'public/build/{}'.format(locale)
+    else:
+        artifact_prefix = 'public/build'
+    manifest_path = "{}/balrog_props.json".format(artifact_prefix)
+    upstream_artifacts.append({
+        "taskId": {"task-reference": taskid_of_manifest},
+        "taskType": "build",
+        "paths": [manifest_path],
+        "locale": locale or "en-US",
+    })
+
+    return upstream_artifacts
+
+
+def generate_build_upstream_artifacts(taskid_to_beetmove, platform, locale=None):
+    upstream_artifacts = generate_upstream_artifacts(taskid_to_beetmove, platform, locale,
+                                                     signing=False)
+    return upstream_artifacts
+
+
+@transforms.add
+def make_task_worker(config, jobs):
+    for job in jobs:
+        valid_beetmover_signing_job = (len(job["dependencies"]) == 2 and
+                                       any(['signing' in j for j in job['dependencies']]))
+        valid_beetmover_build_job = len(job["dependencies"]) == 1
+        if not valid_beetmover_build_job and not valid_beetmover_signing_job:
+            raise NotImplementedError(
+                "beetmover tasks must have either 1 or 2 dependencies. "
+                "If 2, one of those must be a signing task"
+            )
+
+        build_kind = None
+        signing_kind = None
+        locale = job["attributes"].get("locale")
+        platform = job["attributes"]["build_platform"]
+        for dependency in job["dependencies"].keys():
+            if 'signing' in dependency:
+                signing_kind = dependency
+            else:
+                build_kind = dependency
+
+        if signing_kind:
+            taskid_to_beetmove = "<" + str(signing_kind) + ">"
+            taskid_of_manifest = "<" + str(build_kind) + ">"
+            update_manifest = True
+            upstream_artifacts = generate_signing_upstream_artifacts(
+                taskid_to_beetmove, taskid_of_manifest, platform, locale
+            )
+        else:
+            taskid_to_beetmove = "<" + str(build_kind) + ">"
+            update_manifest = False
+            upstream_artifacts = generate_build_upstream_artifacts(
+                taskid_to_beetmove, platform, locale
+            )
+
+        worker = {'implementation': 'beetmover',
+                  'update_manifest': update_manifest,
+                  'upstream-artifacts': upstream_artifacts}
+
+        if locale:
+            worker["locale"] = locale
+
+        job["worker"] = worker
+
+        yield job
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/beetmover_l10n.py
@@ -0,0 +1,43 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the signing task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.treeherder import join_symbol
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def make_beetmover_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+        for locale in dep_job.attributes.get('chunk_locales', []):
+
+            label = job.get('label',
+                            "beetmover-{}-{}".format(locale, dep_job.label))
+
+            group = 'tc-BM-L10n'
+
+            # add the locale code
+            if 'signing' in label:
+                symbol = 'S{}'.format(locale)
+            else:
+                symbol = locale
+
+            treeherder = {
+                'symbol': join_symbol(group, symbol),
+            }
+
+            beet_description = {
+                'dependent-task': dep_job,
+                'treeherder': treeherder,
+                'label': label,
+                'locale': locale,
+            }
+            yield beet_description
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -280,16 +280,43 @@ task_description_schema = Schema({
             Required('taskType'): basestring,
 
             # Paths to the artifacts to sign
             Required('paths'): [basestring],
 
             # Signing formats to use on each of the paths
             Required('formats'): [basestring],
         }],
+    }, {
+        Required('implementation'): 'beetmover',
+
+        # the maximum time to spend signing, in seconds
+        Required('max-run-time', default=600): int,
+
+        # taskid of task with artifacts to beetmove
+        # beetmover template key
+        Required('update_manifest'): bool,
+
+        # locale key, if this is a locale beetmover job
+        Optional('locale'): basestring,
+
+        # list of artifact URLs for the artifacts that should be beetmoved
+        Required('upstream-artifacts'): [{
+            # taskId of the task with the artifact
+            Required('taskId'): taskref_or_string,
+
+            # type of signing task (for CoT)
+            Required('taskType'): basestring,
+
+            # Paths to the artifacts to sign
+            Required('paths'): [basestring],
+
+            # locale is used to map upload path and allow for duplicate simple names
+            Required('locale'): basestring,
+        }],
     }),
 
     # The "when" section contains descriptions of the circumstances
     # under which this task can be "optimized", that is, left out of the
     # task graph because it is unnecessary.
     Optional('when'): Any({
         # This task only needs to be run if a file matching one of the given
         # patterns has changed in the push.  The patterns use the mozpack
@@ -313,16 +340,17 @@ GROUP_NAMES = {
     'tc-T': 'Talos performance tests executed by TaskCluster',
     'tc-T-e10s': 'Talos performance tests executed by TaskCluster with e10s',
     'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
     'tc-W': 'Web platform tests executed by TaskCluster',
     'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
     'tc-X': 'Xpcshell tests executed by TaskCluster',
     'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
     'tc-L10n': 'Localised Repacks executed by Taskcluster',
+    'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
     'Aries': 'Aries Device Image',
     'Nexus 5-L': 'Nexus 5-L Device Image',
     'Cc': 'Toolchain builds',
     'SM-tc': 'Spidermonkey builds',
 }
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
 V2_ROUTE_TEMPLATES = [
@@ -479,16 +507,30 @@ def build_scriptworker_signing_payload(c
     worker = task['worker']
 
     task_def['payload'] = {
         'maxRunTime': worker['max-run-time'],
         'upstreamArtifacts':  worker['upstream-artifacts']
     }
 
 
+@payload_builder('beetmover')
+def build_beetmover_payload(config, task, task_def):
+    worker = task['worker']
+
+    task_def['payload'] = {
+        'maxRunTime': worker['max-run-time'],
+        'upload_date': config.params['build_date'],
+        'update_manifest': worker['update_manifest'],
+        'upstreamArtifacts':  worker['upstream-artifacts']
+    }
+    if worker.get('locale'):
+        task_def['payload']['locale'] = worker['locale']
+
+
 @payload_builder('macosx-engine')
 def build_macosx_engine_payload(config, task, task_def):
     worker = task['worker']
     artifacts = map(lambda artifact: {
         'name': artifact['name'],
         'path': artifact['path'],
         'type': artifact['type'],
         'expires': task_def['expires'],