Bug 1330668 - Schedule balrog submission tasks. r=dustin a=me
authorJustin Wood <Callek@gmail.com>
Thu, 12 Jan 2017 21:01:54 -0500
changeset 329696 98b1156574e311bb8fef5a5f659f32e10d77edfe
parent 329695 96ccea5f9ed9902a7ba4521bc84d136e791017f3
child 329697 7a27e1371c3e6bb8ca6d8000325ab6699d7a8dd1
push id36126
push userkwierso@gmail.com
push dateTue, 17 Jan 2017 21:17:48 +0000
treeherderautoland@98b1156574e3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin, me
bugs1330668
milestone53.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1330668 - Schedule balrog submission tasks. r=dustin a=me MozReview-Commit-ID: ASBDESDps8G
taskcluster/ci/balrog/kind.yml
taskcluster/docs/attributes.rst
taskcluster/docs/kinds.rst
taskcluster/taskgraph/task/balrog.py
taskcluster/taskgraph/transforms/balrog.py
taskcluster/taskgraph/transforms/task.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/balrog/kind.yml
@@ -0,0 +1,13 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+implementation: taskgraph.task.balrog:BalrogTask
+
+transforms:
+   - taskgraph.transforms.balrog:transforms
+   - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - beetmover
+  - beetmover-l10n
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -138,9 +138,9 @@ chunk_locales
 =============
 For the ``l10n`` and ``nightly-l10n`` kinds, this attribute contains an array of
 the individual locales this chunk is responsible for processing.
 
 locale
 ======
 For jobs that operate on only one locale, we set the attribute ``locale`` to the
 specific locale involved. Currently this is only in l10n versions of the
-``beetmover`` kinds.
+``beetmover`` and ``balrog`` kinds.
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -155,16 +155,29 @@ Docker repository
 
 The task definition used to create the image-building tasks is given in
 ``image.yml`` in the kind directory, and is interpreted as a :doc:`YAML
 Template <yaml-templates>`.
 
 android-stuff
 --------------
 
+balrog
+------
+
+Balrog is the Mozilla Update Server. Jobs of this kind are submitting information
+which assists in telling Firefox that an update is available for the related job.
+
+balrog-l10n
+-----------
+
+Balrog is the Mozilla Update Server. Jobs of this kind are submitting information
+which assists in telling Firefox that an update is available for the localized
+job involved.
+
 beetmover
 ---------
 
 Beetmover, takes specific artifacts, "Beets", and pushes them to a location outside
 of Taskcluster's task artifacts, (archive.mozilla.org as one place) and in the
 process determines the final location and a "pretty" name (versioned product name)
 
 beetmover-l10n
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/task/balrog.py
@@ -0,0 +1,28 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from . import transform
+
+
+class BalrogTask(transform.TransformTask):
+    """
+    A task implementing a balrog submission job.  These depend on beetmover jobs
+    and submits the update to balrog as available after the files are moved into place
+    """
+
+    @classmethod
+    def get_inputs(cls, kind, path, config, params, loaded_tasks):
+        if config.get('kind-dependencies', []) != ["beetmover", "beetmover-l10n"]:
+            raise Exception("Balrog kinds must depend on beetmover kinds")
+        for task in loaded_tasks:
+            if not task.attributes.get('nightly'):
+                continue
+            if task.kind not in config.get('kind-dependencies', []):
+                continue
+            beetmover_task = {}
+            beetmover_task['dependent-task'] = task
+
+            yield beetmover_task
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/balrog.py
@@ -0,0 +1,113 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the beetmover task into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import (
+    validate_schema,
+    TransformSequence
+)
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import Schema, Any, Required, Optional
+
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+transforms = TransformSequence()
+
+# shortcut for a string where task references are allowed
+taskref_or_string = Any(
+    basestring,
+    {Required('task-reference'): basestring})
+
+balrog_description_schema = Schema({
+    # the dependent task (object) for this balrog job, used to inform balrogworker.
+    Required('dependent-task'): object,
+
+    # unique label to describe this balrog task, defaults to balrog-{dep.label}
+    Optional('label'): basestring,
+
+    # treeherder is allowed here to override any defaults we use for beetmover.  See
+    # taskcluster/taskgraph/transforms/task.py for the schema details, and the
+    # below transforms for defaults of various values.
+    Optional('treeherder'): task_description_schema['treeherder'],
+})
+
+
+@transforms.add
+def validate(config, jobs):
+    for job in jobs:
+        label = job.get('dependent-task', object).__dict__.get('label', '?no-label?')
+        yield validate_schema(
+            balrog_description_schema, job,
+            "In balrog ({!r} kind) task for {!r}:".format(config.kind, label))
+
+
+@transforms.add
+def skip_unsigned_beets(config, jobs):
+    for job in jobs:
+        if 'signing' not in job['dependent-task'].label:
+            # Skip making a balrog task for this
+            continue
+        yield job
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+
+        treeherder = job.get('treeherder', {})
+        treeherder.setdefault('symbol', 'tc-Up(N)')
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform',
+                              "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('tier', 2)
+        treeherder.setdefault('kind', 'build')
+
+        attributes = {
+            'nightly': dep_job.attributes.get('nightly', False),
+            'build_platform': dep_job.attributes.get('build_platform'),
+            'build_type': dep_job.attributes.get('build_type'),
+        }
+
+        if dep_job.attributes.get('locale'):
+            treeherder['symbol'] = 'tc-Up({})'.format(dep_job.attributes.get('locale'))
+            attributes['locale'] = dep_job.attributes.get('locale')
+
+        label = job.get('label', "balrog-{}".format(dep_job.label))
+
+        upstream_artifacts = [{
+            "taskId": {"task-reference": "<beetmover>"},
+            "taskType": "beetmover",
+            "paths": [
+                "public/manifest.json"
+            ],
+        }]
+
+        task = {
+            'label': label,
+            'description': "{} Balrog".format(
+                dep_job.task["metadata"]["description"]),
+            # do we have to define worker type somewhere?
+            'worker-type': 'scriptworker-prov-v1/balrogworker-v1',
+            'worker': {
+                'implementation': 'balrog',
+                'upstream-artifacts': upstream_artifacts,
+            },
+            # bump this to nightly / release when applicable+permitted
+            'scopes': ["project:releng:balrog:nightly"],
+            'dependencies': {'beetmover': dep_job.label},
+            'attributes': attributes,
+            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'treeherder': treeherder,
+        }
+
+        yield task
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -307,16 +307,30 @@ task_description_schema = Schema({
             Required('taskType'): basestring,
 
             # Paths to the artifacts to sign
             Required('paths'): [basestring],
 
             # locale is used to map upload path and allow for duplicate simple names
             Required('locale'): basestring,
         }],
+    }, {
+        Required('implementation'): 'balrog',
+
+        # list of artifact URLs for the artifacts that should be beetmoved
+        Required('upstream-artifacts'): [{
+            # taskId of the task with the artifact
+            Required('taskId'): taskref_or_string,
+
+            # type of signing task (for CoT)
+            Required('taskType'): basestring,
+
+            # Paths to the artifacts to sign
+            Required('paths'): [basestring],
+        }],
     }),
 
     # The "when" section contains descriptions of the circumstances
     # under which this task can be "optimized", that is, left out of the
     # task graph because it is unnecessary.
     Optional('when'): Any({
         # This task only needs to be run if a file matching one of the given
         # patterns has changed in the push.  The patterns use the mozpack
@@ -341,16 +355,17 @@ GROUP_NAMES = {
     'tc-T-e10s': 'Talos performance tests executed by TaskCluster with e10s',
     'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
     'tc-W': 'Web platform tests executed by TaskCluster',
     'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
     'tc-X': 'Xpcshell tests executed by TaskCluster',
     'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
     'tc-L10n': 'Localised Repacks executed by Taskcluster',
     'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
+    'tc-Up': 'Balrog submission of updates, executed by Taskcluster',
     'Aries': 'Aries Device Image',
     'Nexus 5-L': 'Nexus 5-L Device Image',
     'Cc': 'Toolchain builds',
     'SM-tc': 'Spidermonkey builds',
 }
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
 V2_ROUTE_TEMPLATES = [
@@ -521,16 +536,27 @@ def build_beetmover_payload(config, task
         'upload_date': config.params['build_date'],
         'update_manifest': worker['update_manifest'],
         'upstreamArtifacts':  worker['upstream-artifacts']
     }
     if worker.get('locale'):
         task_def['payload']['locale'] = worker['locale']
 
 
+@payload_builder('balrog')
+def build_balrog_payload(config, task, task_def):
+    worker = task['worker']
+
+    task_def['payload'] = {
+        # signing cert is unused, but required by balrogworker (Bug 1282187 c#7)
+        'signing_cert': "dep",
+        'upstreamArtifacts':  worker['upstream-artifacts']
+    }
+
+
 @payload_builder('macosx-engine')
 def build_macosx_engine_payload(config, task, task_def):
     worker = task['worker']
     artifacts = map(lambda artifact: {
         'name': artifact['name'],
         'path': artifact['path'],
         'type': artifact['type'],
         'expires': task_def['expires'],