Bug 1329282 - Task kind for building QEMU images draft
authorJonas Finnemann Jensen <jopsen@gmail.com>
Thu, 24 Aug 2017 13:05:56 -0700
changeset 664958 897fded81f5d61fd596bda53e6aa2fd722304599
parent 664957 38062cb33d90eb54b7ce37d947076cceca4a59b7
child 664959 261b8ebc5434cb79141480b27e60a056af4b4b61
push id79876
push userjojensen@mozilla.com
push dateThu, 14 Sep 2017 17:51:34 +0000
bugs1329282
milestone57.0a1
Bug 1329282 - Task kind for building QEMU images * Defines a loader for creating a task for each folder * Defines a transform that creates a task which builds and indexes a QEMU image. * Defines a new task kind: qemu-image MozReview-Commit-ID: 7iiCRjktP65
taskcluster/ci/qemu-image/kind.yml
taskcluster/docs/kinds.rst
taskcluster/taskgraph/loader/folders.py
taskcluster/taskgraph/transforms/qemu_image.py
taskcluster/taskgraph/transforms/task.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/qemu-image/kind.yml
@@ -0,0 +1,15 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+loader: taskgraph.loader.folders:loader
+
+transforms:
+  - taskgraph.transforms.qemu_image:transforms
+  - taskgraph.transforms.task:transforms
+
+# taskgraph.loader.folders:loader will create a task for each subfolder in:
+folder: taskcluster/qemu/
+exclude:
+  - qemu-image-builder  # excluding the qemu-image-builder image (which has to be manually built)
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -149,16 +149,22 @@ Docker images are built from subdirector
 ``docker build``.  There is currently no capability for one Docker image to
 depend on another in-tree docker image, without uploading the latter to a
 Docker repository
 
 The task definition used to create the image-building tasks is given in
 ``image.yml`` in the kind directory, and is interpreted as a :doc:`YAML
 Template <yaml-templates>`.
 
+qemu-image
+----------
+
+The ``qemu-image`` task kind builds QEMU images based on in-tree image definitions.
+See ``taskcluster/qemu/`` for images and details.
+
 android-stuff
 -------------
 
 balrog
 ------
 
 Balrog is the Mozilla Update Server. Jobs of this kind are submitting information
 which assists in telling Firefox that an update is available for the related job.
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/loader/folders.py
@@ -0,0 +1,42 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import logging
+
+from .. import GECKO
+
+logger = logging.getLogger(__name__)
+
+
+def loader(kind, path, config, params, loaded_tasks):
+    """
+    Given keys "folder" and "exclude" this loader will create a job for each
+    subfolder in folder (relative to GECKO root), jobs will have the form:
+    ``{name, path}``, where name is the name of the subfolder and path is
+    the absolute path to the folder.
+    """
+    # Validate input
+    folder = config.get('folder')
+    if not isinstance(folder, basestring):
+        raise Exception('taskgraph.loader.folders:loader expects "folder" ' +
+                        'in kind.yml to be a string')
+    exclude = config.get('exclude', [])
+    if not isinstance(exclude, list) or not all((isinstance(s, basestring) for s in exclude)):
+        raise Exception('taskgraph.loader.folders:loader expects "exclude" ' +
+                        'in kind.yml to be a list of strings')
+
+    root_folder = os.path.join(GECKO, folder)
+    for entry in os.listdir(root_folder):
+        if not os.path.isdir(os.path.join(root_folder, entry)):
+            continue  # Skip non-folders
+        if entry in exclude:
+            continue  # Skip excluded folders
+        # Create job for subfolder
+        yield {
+            'name': entry,
+            'folder': os.path.join(root_folder, entry),
+        }
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/qemu_image.py
@@ -0,0 +1,107 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform {name, path} entries to task description template for QEMU images.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+from taskgraph.qemu.imagehash import compute_image_hash
+from taskgraph.qemu.imageschema import (
+    load_and_validate_image,
+    load_reference,
+    has_image,
+    has_reference,
+)
+from taskgraph.qemu.imagecache import (
+    PRIVATE_ARTIFACT_NAME,
+    PUBLIC_ARTIFACT_NAME,
+    index_namespace,
+    auxiliary_index_namespaces,
+)
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def create_qemu_image_task(config, tasks):
+    """
+    create tasks for building qemu images
+    """
+    # Find reference to qemu-image-builder image
+    builder_ref = load_reference('qemu-image-builder')
+
+    # For each folder on the form {name, path}
+    for folder in tasks:
+        image_name = folder['name']
+
+        # Skip folders that don't have image.yml or has a reference.yml
+        if not has_image(image_name) or has_reference(image_name):
+            continue
+
+        image = load_and_validate_image(image_name)
+        image_hash = compute_image_hash(image_name)
+
+        # Construct index namespaces
+        namespaces = [
+            index_namespace(image_name, image_hash, config.params['level']),
+        ] + auxiliary_index_namespaces(
+            image_name,
+            config.params['level'],
+            config.params['moz_build_date'],
+        )
+
+        # Allow it to be replaced with a level that is higher than or equal to
+        # current level, always prefer higher levels to improve caching.
+        optimizations = [[
+            'index-search',
+            index_namespace(image_name, image_hash, level)
+        ] for level in reversed(range(int(config.params['level']), 4))]
+
+        yield {
+            'label': 'build-qemu-image-{}'.format(image_name),
+            'description': 'Build QEMU image: `{}` from in-tree recipe\n\n---\n{}'.format(
+                image_name, image['description'],
+            ),
+            'attributes': {'image_name': image_name},
+            'expires-after': '1 year',
+            'routes': ['index.{}'.format(ns) for ns in namespaces],
+            'optimizations': optimizations,
+            'scopes': ['secrets:get:project/taskcluster/gecko/hgfingerprint'],
+            'treeherder': {
+                'symbol': image['symbol'],
+                'platform': 'taskcluster-images/opt',
+                'kind': 'other',
+                'tier': 1,
+            },
+            'run-on-projects': [],
+            'worker-type': 'manual-packet/tc-worker-qemu-v1',
+            # can't use {in-tree: ..} here, otherwise we might try to build
+            # this image..
+            'worker': {
+                'implementation': 'taskcluster-worker:qemu-engine',
+                'image': {
+                    'url': builder_ref['url'],
+                    'sha256': builder_ref['sha256'],
+                },
+                'command': [
+                    'run-task', '--vcs-checkout=/home/worker/checkouts/gecko', '--',
+                    '/home/worker/checkouts/gecko/mach', 'qemu', 'build',
+                    image_name, '--output', '/home/worker/image.tar.zst',
+                ],
+                'env': {
+                    'GECKO_BASE_REPOSITORY': config.params['base_repository'],
+                    'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
+                    'GECKO_HEAD_REV': config.params['head_rev'],
+                },
+                'artifacts': [{
+                    'type': 'file',
+                    'name': PRIVATE_ARTIFACT_NAME if image['private'] else PUBLIC_ARTIFACT_NAME,
+                    'path': '/home/worker/image.tar.zst',
+                }],
+                'max-run-time': '3 hours',
+            },
+        }
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -22,16 +22,18 @@ from taskgraph.util.attributes import TR
 from taskgraph.util.hash import hash_path
 from taskgraph.util.treeherder import split_symbol
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import validate_schema, Schema
 from taskgraph.util.scriptworker import get_release_config
 from voluptuous import Any, Required, Optional, Extra
 from taskgraph import GECKO
 from ..util import docker as dockerutil
+import taskgraph.qemu.imageschema
+import taskgraph.qemu.imagecache
 
 from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR
 
 
 RUN_TASK = os.path.join(GECKO, 'taskcluster', 'docker', 'recipes', 'run-task')
 
 
 @memoize
@@ -371,16 +373,47 @@ task_description_schema = Schema({
             # task image path from which to read artifact
             Required('path'): basestring,
 
             # name of the produced artifact (root of the names for
             # type=directory)
             Required('name'): basestring,
         }],
     }, {
+        Required('implementation'): 'taskcluster-worker:qemu-engine',
+
+        # QEMU image reference
+        Required('image'): Any(
+            basestring,  # URL to image
+            {'in-tree': basestring},  # in-tree image by name
+            {'url': basestring, 'sha256': basestring},  # url and sha256
+        ),
+
+        # the command to run
+        Required('command'): [basestring],
+
+        # environment variables
+        Required('env'): {basestring: taskref_or_string},
+
+        # Maximum runtime
+        Required('max-run-time'): basestring,
+
+        # artifacts to extract from the task image after completion
+        Optional('artifacts'): [{
+            # type of artifact -- simple file, or recursive directory
+            Required('type'): Any('file', 'directory'),
+
+            # task image path from which to read artifact
+            Required('path'): basestring,
+
+            # name of the produced artifact (root of the names for
+            # type=directory)
+            Required('name'): basestring,
+        }],
+    }, {
         Required('implementation'): 'scriptworker-signing',
 
         # the maximum time to spend signing, in seconds
         Required('max-run-time', default=600): int,
 
         # list of artifact URLs for the artifacts that should be signed
         Required('upstream-artifacts'): [{
             # taskId of the task with the artifact
@@ -920,16 +953,53 @@ def build_macosx_engine_payload(config, 
     }
     if worker.get('reboot'):
         task_def['payload'] = worker['reboot']
 
     if task.get('needs-sccache'):
         raise Exception('needs-sccache not supported in native-engine')
 
 
+@payload_builder('taskcluster-worker:qemu-engine')
+def build_taskcluster_worker_qemu_engine_payload(config, task, task_def):
+    worker = task['worker']
+    artifacts = map(lambda artifact: {
+        'name': artifact['name'],
+        'path': artifact['path'],
+        'type': artifact['type'],
+        'expires': task_def['expires'],
+    }, worker.get('artifacts', []))
+    image = worker['image']
+    if isinstance(image, dict) and 'in-tree' in image:
+        image_name = image['in-tree']
+        ref = taskgraph.qemu.imageschema.load_reference(image_name)
+        if ref:
+            image = {
+                'url': ref['url'],
+                'sha256': ref['sha256'],
+            }
+        else:
+            img = taskgraph.qemu.imageschema.load_and_validate_image(image_name)
+            task.setdefault('dependencies', {})['qemu-image'] = 'build-qemu-image-' + image_name
+            artifact = taskgraph.qemu.imagecache.PUBLIC_ARTIFACT_NAME
+            if img['private']:
+                artifact = taskgraph.qemu.imagecache.PRIVATE_ARTIFACT_NAME
+            image = {
+                'taskId': {'task-reference': 'qemu-image'},
+                'artifact': artifact,
+            }
+    task_def['payload'] = {
+        'image': image,
+        'command': worker['command'],
+        'env': worker['env'],
+        'artifacts': artifacts,
+        'maxRunTime': worker['max-run-time'],
+    }
+
+
 @payload_builder('buildbot-bridge')
 def build_buildbot_bridge_payload(config, task, task_def):
     del task['extra']['treeherder']
     del task['extra']['treeherderEnv']
     worker = task['worker']
     task_def['payload'] = {
         'buildername': worker['buildername'],
         'sourcestamp': worker['sourcestamp'],