Bug 1330668 - Schedule balrog submission tasks. r=dustin a=me
authorJustin Wood <Callek@gmail.com>
Thu, 12 Jan 2017 21:01:54 -0500
changeset 357711 98b1156574e311bb8fef5a5f659f32e10d77edfe
parent 357710 96ccea5f9ed9902a7ba4521bc84d136e791017f3
child 357712 7a27e1371c3e6bb8ca6d8000325ab6699d7a8dd1
push id10621
push userjlund@mozilla.com
push dateMon, 23 Jan 2017 16:02:43 +0000
treeherdermozilla-aurora@dca7b42e6c67 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin, me
bugs1330668
milestone53.0a1
Bug 1330668 - Schedule balrog submission tasks. r=dustin a=me MozReview-Commit-ID: ASBDESDps8G
taskcluster/ci/balrog/kind.yml
taskcluster/docs/attributes.rst
taskcluster/docs/kinds.rst
taskcluster/taskgraph/task/balrog.py
taskcluster/taskgraph/transforms/balrog.py
taskcluster/taskgraph/transforms/task.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/balrog/kind.yml
@@ -0,0 +1,13 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.balrog:BalrogTask
+
+transforms:
+   - taskgraph.transforms.balrog:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - beetmover
+  - beetmover-l10n
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -138,9 +138,9 @@ chunk_locales
 =============
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains an array of
 the individual locales this chunk is responsible for processing.
 
 locale
 ======
 For jobs that operate on only one locale, we set the attribute ``locale`` to the
 specific locale involved. Currently this is only in l10n versions of the
-``beetmover`` kinds.
+``beetmover`` and ``balrog`` kinds.
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -155,16 +155,29 @@ Docker repository
 
 The task definition used to create the image-building tasks is given in
 ``image.yml`` in the kind directory, and is interpreted as a :doc:`YAML
 Template <yaml-templates>`.
 
 android-stuff
 --------------
 
+balrog
+------
+
+Balrog is the Mozilla Update Server. Jobs of this kind are submitting information
+which assists in telling Firefox that an update is available for the related job.
+
+balrog-l10n
+-----------
+
+Balrog is the Mozilla Update Server. Jobs of this kind are submitting information
+which assists in telling Firefox that an update is available for the localized
+job involved.
+
 beetmover
 ---------
 
 Beetmover, takes specific artifacts, "Beets", and pushes them to a location outside
 of Taskcluster's task artifacts, (archive.mozilla.org as one place) and in the
 process determines the final location and a "pretty" name (versioned product name)
 
 beetmover-l10n
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/task/balrog.py
@@ -0,0 +1,28 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from . import transform
+
+
+class BalrogTask(transform.TransformTask):
+    """
+    A task implementing a balrog submission job.  These depend on beetmover jobs
+    and submits the update to balrog as available after the files are moved into place
+    """
+
+    @classmethod
+    def get_inputs(cls, kind, path, config, params, loaded_tasks):
+        if config.get('kind-dependencies', []) != ["beetmover", "beetmover-l10n"]:
+            raise Exception("Balrog kinds must depend on beetmover kinds")
+        for task in loaded_tasks:
+            if not task.attributes.get('nightly'):
+                continue
+            if task.kind not in config.get('kind-dependencies', []):
+                continue
+            beetmover_task = {}
+            beetmover_task['dependent-task'] = task
+
+            yield beetmover_task
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/balrog.py
@@ -0,0 +1,113 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import (
+    validate_schema,
+    TransformSequence
+)
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Schema, Any, Required, Optional
+
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+transforms = TransformSequence()
+
+# shortcut for a string where task references are allowed
+taskref_or_string = Any(
+    basestring,
+    {Required('task-reference'): basestring})
+
+balrog_description_schema = Schema({
+    # the dependent task (object) for this balrog job, used to inform balrogworker.
+    Required('dependent-task'): object,
+
+    # unique label to describe this balrog task, defaults to balrog-{dep.label}
+    Optional('label'): basestring,
+
+    # treeherder is allowed here to override any defaults we use for beetmover.  See
+    # taskcluster/taskgraph/transforms/task.py for the schema details, and the
+    # below transforms for defaults of various values.
+    Optional('treeherder'): task_description_schema['treeherder'],
+})
+
+
+@transforms.add
+def validate(config, jobs):
+    for job in jobs:
+        label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+        yield validate_schema(
+            balrog_description_schema, job,
+            "In balrog ({!r} kind) task for {!r}:".format(config.kind, label))
+
+
+@transforms.add
+def skip_unsigned_beets(config, jobs):
+    for job in jobs:
+        if 'signing' not in job['dependent-task'].label:
+            # Skip making a balrog task for this
+            continue
+        yield job
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+
+        treeherder = job.get('treeherder', {})
+        treeherder.setdefault('symbol', 'tc-Up(N)')
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform',
+                              "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('tier', 2)
+        treeherder.setdefault('kind', 'build')
+
+        attributes = {
+            'nightly': dep_job.attributes.get('nightly', False),
+            'build_platform': dep_job.attributes.get('build_platform'),
+            'build_type': dep_job.attributes.get('build_type'),
+        }
+
+        if dep_job.attributes.get('locale'):
+            treeherder['symbol'] = 'tc-Up({})'.format(dep_job.attributes.get('locale'))
+            attributes['locale'] = dep_job.attributes.get('locale')
+
+        label = job.get('label', "balrog-{}".format(dep_job.label))
+
+        upstream_artifacts = [{
+            "taskId": {"task-reference": "<beetmover>"},
+            "taskType": "beetmover",
+            "paths": [
+                "public/manifest.json"
+            ],
+        }]
+
+        task = {
+            'label': label,
+            'description': "{} Balrog".format(
+                dep_job.task["metadata"]["description"]),
+            # do we have to define worker type somewhere?
+            'worker-type': 'scriptworker-prov-v1/balrogworker-v1',
+            'worker': {
+                'implementation': 'balrog',
+                'upstream-artifacts': upstream_artifacts,
+            },
+            # bump this to nightly / release when applicable+permitted
+            'scopes': ["project:releng:balrog:nightly"],
+            'dependencies': {'beetmover': dep_job.label},
+            'attributes': attributes,
+            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'treeherder': treeherder,
+        }
+
+        yield task
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -307,16 +307,30 @@ task_description_schema = Schema({
             Required('taskType'): basestring,
 
             # Paths to the artifacts to sign
             Required('paths'): [basestring],
 
             # locale is used to map upload path and allow for duplicate simple names
             Required('locale'): basestring,
         }],
+    }, {
+        Required('implementation'): 'balrog',
+
+        # list of artifact URLs for the artifacts that should be beetmoved
+        Required('upstream-artifacts'): [{
+            # taskId of the task with the artifact
+            Required('taskId'): taskref_or_string,
+
+            # type of signing task (for CoT)
+            Required('taskType'): basestring,
+
+            # Paths to the artifacts to sign
+            Required('paths'): [basestring],
+        }],
     }),
 
     # The "when" section contains descriptions of the circumstances
     # under which this task can be "optimized", that is, left out of the
     # task graph because it is unnecessary.
     Optional('when'): Any({
         # This task only needs to be run if a file matching one of the given
         # patterns has changed in the push.  The patterns use the mozpack
@@ -341,16 +355,17 @@ GROUP_NAMES = {
     'tc-T-e10s': 'Talos performance tests executed by TaskCluster with e10s',
     'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
     'tc-W': 'Web platform tests executed by TaskCluster',
     'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
     'tc-X': 'Xpcshell tests executed by TaskCluster',
     'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
     'tc-L10n': 'Localised Repacks executed by Taskcluster',
     'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
+    'tc-Up': 'Balrog submission of updates, executed by Taskcluster',
     'Aries': 'Aries Device Image',
     'Nexus 5-L': 'Nexus 5-L Device Image',
     'Cc': 'Toolchain builds',
     'SM-tc': 'Spidermonkey builds',
 }
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
 V2_ROUTE_TEMPLATES = [
@@ -521,16 +536,27 @@ def build_beetmover_payload(config, task
         'upload_date': config.params['build_date'],
         'update_manifest': worker['update_manifest'],
         'upstreamArtifacts':  worker['upstream-artifacts']
     }
     if worker.get('locale'):
         task_def['payload']['locale'] = worker['locale']
 
 
+@payload_builder('balrog')
+def build_balrog_payload(config, task, task_def):
+    worker = task['worker']
+
+    task_def['payload'] = {
+        # signing cert is unused, but required by balrogworker (Bug 1282187 c#7)
+        'signing_cert': "dep",
+        'upstreamArtifacts':  worker['upstream-artifacts']
+    }
+
+
 @payload_builder('macosx-engine')
 def build_macosx_engine_payload(config, task, task_def):
     worker = task['worker']
     artifacts = map(lambda artifact: {
         'name': artifact['name'],
         'path': artifact['path'],
         'type': artifact['type'],
         'expires': task_def['expires'],