Bug 1330668 - Schedule beetmover tasks. r=dustin
authorJustin Wood <Callek@gmail.com>
Thu, 12 Jan 2017 17:45:36 -0500
changeset 374685 028c582db1762776cfa03f91805f74ba5e4453a5
parent 374684 023c679970910012d191a131218414220ed19121
child 374686 96ccea5f9ed9902a7ba4521bc84d136e791017f3
push id6996
push userjlorenzo@mozilla.com
push dateMon, 06 Mar 2017 20:48:21 +0000
treeherdermozilla-beta@d89512dab048 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1330668
milestone53.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1330668 - Schedule beetmover tasks. r=dustin ToDo: attribute this cset to changes on date. MozReview-Commit-ID: 7gl3aLGNWvP
taskcluster/ci/beetmover-l10n/kind.yml
taskcluster/ci/beetmover/kind.yml
taskcluster/docs/attributes.rst
taskcluster/docs/kinds.rst
taskcluster/taskgraph/task/beetmover.py
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/beetmover_l10n.py
taskcluster/taskgraph/transforms/task.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/beetmover-l10n/kind.yml
@@ -0,0 +1,14 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.beetmover:BeetmoverTask
+
+transforms:
+   - taskgraph.transforms.beetmover_l10n:transforms
+   - taskgraph.transforms.beetmover:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - nightly-l10n
+  - nightly-l10n-signing
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/beetmover/kind.yml
@@ -0,0 +1,13 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.beetmover:BeetmoverTask
+
+transforms:
+   - taskgraph.transforms.beetmover:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - build
+  - build-signing
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -133,8 +133,14 @@ l10n_chunk
 ==========
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains the chunk
 number of the job. Note that this is a string!
 
 chunk_locales
 =============
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains an array of
 the individual locales this chunk is responsible for processing.
+
+locale
+======
+For jobs that operate on only one locale, we set the attribute ``locale`` to the
+specific locale involved. Currently this is only in l10n versions of the
+``beetmover`` kinds.
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -154,8 +154,24 @@ depend on another in-tree docker image, 
 Docker repository
 
 The task definition used to create the image-building tasks is given in
 ``image.yml`` in the kind directory, and is interpreted as a :doc:`YAML
 Template <yaml-templates>`.
 
 android-stuff
 --------------
+
+beetmover
+---------
+
+Beetmover, takes specific artifacts, "Beets", and pushes them to a location outside
+of Taskcluster's task artifacts, (archive.mozilla.org as one place) and in the
+process determines the final location and a "pretty" name (versioned product name)
+
+beetmover-l10n
+--------------
+
+Beetmover L10n, takes specific artifacts, "Beets", and pushes them to a location outside
+of Taskcluster's task artifacts, (archive.mozilla.org as one place) and in the
+process determines the final location and a "pretty" name (versioned product name)
+This separate kind uses logic specific to localized artifacts, such as including
+the language in the final artifact names.
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/task/beetmover.py
@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from . import transform
+
+
+class BeetmoverTask(transform.TransformTask):
+    """
+    A task implementing a signing job.  These depend on nightly build jobs and
+    sign the artifacts after a build has completed.
+
+    We use a dictionary to create the input to the transforms.
+    It will have added to it keys `build-label`, the label for the build task,
+    and `build-platform` / `build-type`, its platform and type.
+    """
+
+    @classmethod
+    def get_inputs(cls, kind, path, config, params, loaded_tasks):
+        if config.get('kind-dependencies', []) != ["build", "build-signing"] and \
+           config.get('kind-dependencies', []) != ["nightly-l10n", "nightly-l10n-signing"]:
+            raise Exception("Beetmover kinds must depend on builds or signing builds")
+        for task in loaded_tasks:
+            if not task.attributes.get('nightly'):
+                continue
+            if task.kind not in config.get('kind-dependencies'):
+                continue
+            beetmover_task = {}
+            beetmover_task['dependent-task'] = task
+
+            yield beetmover_task
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -0,0 +1,330 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import (
+    validate_schema,
+    TransformSequence
+)
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Schema, Any, Required, Optional
+
+
+_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
+    "balrog_props.json",
+    "target.common.tests.zip",
+    "target.cppunittest.tests.zip",
+    "target.crashreporter-symbols.zip",
+    "target.json",
+    "target.mochitest.tests.zip",
+    "target.mozinfo.json",
+    "target.reftest.tests.zip",
+    "target.talos.tests.zip",
+    "target.test_packages.json",
+    "target.txt",
+    "target.web-platform.tests.zip",
+    "target.xpcshell.tests.zip",
+    "target_info.txt",
+    "target.jsshell.zip",
+    "mozharness.zip",
+    "target.langpack.xpi",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US = [
+    "update/target.complete.mar",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N = [
+    "target.langpack.xpi",
+    "balrog_props.json",
+]
+_DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N = [
+    "target.complete.mar",
+]
+_MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US = [
+    "en-US/target.common.tests.zip",
+    "en-US/target.cppunittest.tests.zip",
+    "en-US/target.json",
+    "en-US/target.mochitest.tests.zip",
+    "en-US/target.mozinfo.json",
+    "en-US/target.reftest.tests.zip",
+    "en-US/target.talos.tests.zip",
+    "en-US/target.test_packages.json",
+    "en-US/target.txt",
+    "en-US/target.web-platform.tests.zip",
+    "en-US/target.xpcshell.tests.zip",
+    "en-US/target_info.txt",
+    "en-US/bouncer.apk",
+    "en-US/mozharness.zip",
+    "en-US/robocop.apk",
+    "en-US/target.jsshell.zip",
+]
+_MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI = [
+    "balrog_props.json",
+    "target.common.tests.zip",
+    "target.cppunittest.tests.zip",
+    "target.json",
+    "target.mochitest.tests.zip",
+    "target.mozinfo.json",
+    "target.reftest.tests.zip",
+    "target.talos.tests.zip",
+    "target.test_packages.json",
+    "target.txt",
+    "target.web-platform.tests.zip",
+    "target.xpcshell.tests.zip",
+    "target_info.txt",
+    "bouncer.apk",
+    "mozharness.zip",
+    "robocop.apk",
+    "target.jsshell.zip",
+]
+_MOBILE_UPSTREAM_ARTIFACTS_SIGNED_EN_US = [
+    "en-US/target.apk",
+]
+_MOBILE_UPSTREAM_ARTIFACTS_SIGNED_MULTI = [
+    "target.apk",
+]
+
+
+UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
+    'linux64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
+        "target.sdk.tar.bz2"
+    ],
+    'linux-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
+        "target.sdk.tar.bz2"
+    ],
+    'android-x86-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
+    'android-api-15-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
+    'macosx64-nightly': [],
+
+    'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
+    'linux-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
+    'android-x86-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
+    'android-api-15-nightly-l10n': ["balrog_props.json"],
+    'android-api-15-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
+    'macosx64-nightly-l10n': [],
+}
+UPSTREAM_ARTIFACT_SIGNED_PATHS = {
+    'linux64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
+        "target.tar.bz2",
+        "target.tar.bz2.asc",
+    ],
+    'linux-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
+        "target.tar.bz2",
+        "target.tar.bz2.asc",
+    ],
+    'android-x86-nightly': ["en-US/target.apk"],
+    'android-api-15-nightly': ["en-US/target.apk"],
+    'macosx64-nightly': [],
+
+    'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N + [
+        "target.tar.bz2",
+        "target.tar.bz2.asc",
+    ],
+    'linux-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N + [
+        "target.tar.bz2",
+        "target.tar.bz2.asc",
+    ],
+    'android-x86-nightly-multi': ["target.apk"],
+    'android-api-15-nightly-l10n': ["target.apk"],
+    'android-api-15-nightly-multi': ["target.apk"],
+    'macosx64-nightly-l10n': [],
+}
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+transforms = TransformSequence()
+
+# shortcut for a string where task references are allowed
+taskref_or_string = Any(
+    basestring,
+    {Required('task-reference'): basestring})
+
+beetmover_description_schema = Schema({
+    # the dependent task (object) for this beetmover job, used to inform beetmover.
+    Required('dependent-task'): object,
+
+    # depname is used in taskref's to identify the taskID of the unsigned things
+    Required('depname', default='build'): basestring,
+
+    # unique label to describe this beetmover task, defaults to {dep.label}-beetmover
+    Optional('label'): basestring,
+
+    # treeherder is allowed here to override any defaults we use for beetmover.  See
+    # taskcluster/taskgraph/transforms/task.py for the schema details, and the
+    # below transforms for defaults of various values.
+    Optional('treeherder'): task_description_schema['treeherder'],
+
+    # locale is passed only for l10n beetmoving
+    Optional('locale'): basestring,
+})
+
+
+@transforms.add
+def validate(config, jobs):
+    for job in jobs:
+        label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+        yield validate_schema(
+            beetmover_description_schema, job,
+            "In beetmover ({!r} kind) task for {!r}:".format(config.kind, label))
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+
+        treeherder = job.get('treeherder', {})
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform',
+                              "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('tier', 2)
+        treeherder.setdefault('kind', 'build')
+
+        label = job.get('label', "beetmover-{}".format(dep_job.label))
+        dependent_kind = str(dep_job.kind)
+        dependencies = {dependent_kind: dep_job.label}
+        # taskid_of_manifest always refers to the unsigned task
+        if "signing" in dependent_kind:
+            if len(dep_job.dependencies) > 1:
+                raise NotImplementedError(
+                    "can't beetmove a signing task with multiple dependencies")
+            signing_dependencies = dep_job.dependencies
+            dependencies.update(signing_dependencies)
+            treeherder.setdefault('symbol', 'tc(BM-S)')
+        else:
+            treeherder.setdefault('symbol', 'tc(BM)')
+
+        attributes = {
+                'nightly': dep_job.attributes.get('nightly', False),
+                'build_platform': dep_job.attributes.get('build_platform'),
+                'build_type': dep_job.attributes.get('build_type'),
+        }
+        if job.get('locale'):
+            attributes['locale'] = job['locale']
+
+        task = {
+            'label': label,
+            'description': "{} Beetmover".format(
+                dep_job.task["metadata"]["description"]),
+            # do we have to define worker type somewhere?
+            'worker-type': 'scriptworker-prov-v1/beetmoverworker-v1',
+            # bump this to nightly / release when applicable+permitted
+            'scopes': ["project:releng:beetmover:nightly"],
+            'dependencies': dependencies,
+            'attributes': attributes,
+            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'treeherder': treeherder,
+        }
+
+        yield task
+
+
+def generate_upstream_artifacts(taskid_to_beetmove, platform, locale=None, signing=False):
+    task_type = "build"
+    mapping = UPSTREAM_ARTIFACT_UNSIGNED_PATHS
+    if signing:
+        task_type = "signing"
+        mapping = UPSTREAM_ARTIFACT_SIGNED_PATHS
+
+    artifact_prefix = 'public/build'
+    if locale:
+        artifact_prefix = 'public/build/{}'.format(locale)
+        platform = "{}-l10n".format(platform)
+
+    upstream_artifacts = [{
+        "taskId": {"task-reference": taskid_to_beetmove},
+        "taskType": task_type,
+        "paths": ["{}/{}".format(artifact_prefix, p) for p in mapping[platform]],
+        "locale": locale or "en-US",
+    }]
+    if not locale and "android" in platform:
+        # edge case to support 'multi' locale paths
+        multi_platform = "{}-multi".format(platform)
+        upstream_artifacts.append({
+            "taskId": {"task-reference": taskid_to_beetmove},
+            "taskType": task_type,
+            "paths": ["{}/{}".format(artifact_prefix, p) for p in mapping[multi_platform]],
+            "locale": "multi",
+        })
+
+    return upstream_artifacts
+
+
+def generate_signing_upstream_artifacts(taskid_to_beetmove, taskid_of_manifest, platform,
+                                        locale=None):
+    upstream_artifacts = generate_upstream_artifacts(taskid_to_beetmove, platform, locale,
+                                                     signing=True)
+    if locale:
+        artifact_prefix = 'public/build/{}'.format(locale)
+    else:
+        artifact_prefix = 'public/build'
+    manifest_path = "{}/balrog_props.json".format(artifact_prefix)
+    upstream_artifacts.append({
+        "taskId": {"task-reference": taskid_of_manifest},
+        "taskType": "build",
+        "paths": [manifest_path],
+        "locale": locale or "en-US",
+    })
+
+    return upstream_artifacts
+
+
+def generate_build_upstream_artifacts(taskid_to_beetmove, platform, locale=None):
+    upstream_artifacts = generate_upstream_artifacts(taskid_to_beetmove, platform, locale,
+                                                     signing=False)
+    return upstream_artifacts
+
+
+@transforms.add
+def make_task_worker(config, jobs):
+    for job in jobs:
+        valid_beetmover_signing_job = (len(job["dependencies"]) == 2 and
+                                       any(['signing' in j for j in job['dependencies']]))
+        valid_beetmover_build_job = len(job["dependencies"]) == 1
+        if not valid_beetmover_build_job and not valid_beetmover_signing_job:
+            raise NotImplementedError(
+                "beetmover tasks must have either 1 or 2 dependencies. "
+                "If 2, one of those must be a signing task"
+            )
+
+        build_kind = None
+        signing_kind = None
+        locale = job["attributes"].get("locale")
+        platform = job["attributes"]["build_platform"]
+        for dependency in job["dependencies"].keys():
+            if 'signing' in dependency:
+                signing_kind = dependency
+            else:
+                build_kind = dependency
+
+        if signing_kind:
+            taskid_to_beetmove = "<" + str(signing_kind) + ">"
+            taskid_of_manifest = "<" + str(build_kind) + ">"
+            update_manifest = True
+            upstream_artifacts = generate_signing_upstream_artifacts(
+                taskid_to_beetmove, taskid_of_manifest, platform, locale
+            )
+        else:
+            taskid_to_beetmove = "<" + str(build_kind) + ">"
+            update_manifest = False
+            upstream_artifacts = generate_build_upstream_artifacts(
+                taskid_to_beetmove, platform, locale
+            )
+
+        worker = {'implementation': 'beetmover',
+                  'update_manifest': update_manifest,
+                  'upstream-artifacts': upstream_artifacts}
+
+        if locale:
+            worker["locale"] = locale
+
+        job["worker"] = worker
+
+        yield job
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/beetmover_l10n.py
@@ -0,0 +1,43 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the signing task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.treeherder import join_symbol
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def make_beetmover_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+        for locale in dep_job.attributes.get('chunk_locales', []):
+
+            label = job.get('label',
+                            "beetmover-{}-{}".format(locale, dep_job.label))
+
+            group = 'tc-BM-L10n'
+
+            # add the locale code
+            if 'signing' in label:
+                symbol = 'S{}'.format(locale)
+            else:
+                symbol = locale
+
+            treeherder = {
+                'symbol': join_symbol(group, symbol),
+            }
+
+            beet_description = {
+                'dependent-task': dep_job,
+                'treeherder': treeherder,
+                'label': label,
+                'locale': locale,
+            }
+            yield beet_description
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -280,16 +280,43 @@ task_description_schema = Schema({
             Required('taskType'): basestring,
 
             # Paths to the artifacts to sign
             Required('paths'): [basestring],
 
             # Signing formats to use on each of the paths
             Required('formats'): [basestring],
         }],
+    }, {
+        Required('implementation'): 'beetmover',
+
+        # the maximum time to spend signing, in seconds
+        Required('max-run-time', default=600): int,
+
+        # taskid of task with artifacts to beetmove
+        # beetmover template key
+        Required('update_manifest'): bool,
+
+        # locale key, if this is a locale beetmover job
+        Optional('locale'): basestring,
+
+        # list of artifact URLs for the artifacts that should be beetmoved
+        Required('upstream-artifacts'): [{
+            # taskId of the task with the artifact
+            Required('taskId'): taskref_or_string,
+
+            # type of signing task (for CoT)
+            Required('taskType'): basestring,
+
+            # Paths to the artifacts to sign
+            Required('paths'): [basestring],
+
+            # locale is used to map upload path and allow for duplicate simple names
+            Required('locale'): basestring,
+        }],
     }),
 
     # The "when" section contains descriptions of the circumstances
     # under which this task can be "optimized", that is, left out of the
     # task graph because it is unnecessary.
     Optional('when'): Any({
         # This task only needs to be run if a file matching one of the given
         # patterns has changed in the push.  The patterns use the mozpack
@@ -313,16 +340,17 @@ GROUP_NAMES = {
     'tc-T': 'Talos performance tests executed by TaskCluster',
     'tc-T-e10s': 'Talos performance tests executed by TaskCluster with e10s',
     'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
     'tc-W': 'Web platform tests executed by TaskCluster',
     'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
     'tc-X': 'Xpcshell tests executed by TaskCluster',
     'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
     'tc-L10n': 'Localised Repacks executed by Taskcluster',
+    'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
     'Aries': 'Aries Device Image',
     'Nexus 5-L': 'Nexus 5-L Device Image',
     'Cc': 'Toolchain builds',
     'SM-tc': 'Spidermonkey builds',
 }
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
 V2_ROUTE_TEMPLATES = [
@@ -479,16 +507,30 @@ def build_scriptworker_signing_payload(c
     worker = task['worker']
 
     task_def['payload'] = {
         'maxRunTime': worker['max-run-time'],
         'upstreamArtifacts':  worker['upstream-artifacts']
     }
 
 
+@payload_builder('beetmover')
+def build_beetmover_payload(config, task, task_def):
+    worker = task['worker']
+
+    task_def['payload'] = {
+        'maxRunTime': worker['max-run-time'],
+        'upload_date': config.params['build_date'],
+        'update_manifest': worker['update_manifest'],
+        'upstreamArtifacts':  worker['upstream-artifacts']
+    }
+    if worker.get('locale'):
+        task_def['payload']['locale'] = worker['locale']
+
+
 @payload_builder('macosx-engine')
 def build_macosx_engine_payload(config, task, task_def):
     worker = task['worker']
     artifacts = map(lambda artifact: {
         'name': artifact['name'],
         'path': artifact['path'],
         'type': artifact['type'],
         'expires': task_def['expires'],