Bug 1336559 - Add ability to depend on build artifacts to 'run_task' based tasks, r=dustin
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Fri, 24 Feb 2017 09:04:22 -0500
changeset 373779 0ed19e152444a6bd9c45d6ebd485cf5ef47ba46c
parent 373778 23fbbffc07270141b7069876e62f0d4f88c00095
child 373780 3fcbccbc04b8f989f09536266a3a30b8838149f8
push id10863
push userjlorenzo@mozilla.com
push dateMon, 06 Mar 2017 23:02:23 +0000
treeherdermozilla-aurora@0931190cd725 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1336559
milestone54.0a1
Bug 1336559 - Add ability to depend on build artifacts to 'run_task' based tasks, r=dustin Currently 'run_task' tasks have no easy way to depend on a build task. For example, some python unittests need a Firefox binary for their tests, like the mozrunner tests and future test harness selftests (like mochitest tests). This patch allows kinds to add a new key to the kind config which maps test platforms to build-labels. Then 'run_task' jobs can add a 'requires-build': true field to get a build dependency automatically added. The build artifact url will also be stored in the $GECKO_INSTALLER_URL environment variable on the test host. MozReview-Commit-ID: Jqyhbj7nC6z
taskcluster/ci/source-test/kind.yml
taskcluster/taskgraph/transforms/job/__init__.py
taskcluster/taskgraph/transforms/job/common.py
taskcluster/taskgraph/transforms/job/mach.py
taskcluster/taskgraph/transforms/job/run_task.py
--- a/taskcluster/ci/source-test/kind.yml
+++ b/taskcluster/ci/source-test/kind.yml
@@ -9,8 +9,13 @@ transforms:
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 jobs-from:
     - python-tests.yml
     - mozlint.yml
     - doc.yml
     - webidl.yml
+
+# This is used by run-task based tasks to lookup which build task it
+# should depend on based on its own platform.
+dependent-build-platforms:
+    linux64.*: build-linux64/opt
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -128,35 +128,35 @@ def make_task_description(config, jobs):
     import_all()
     for job in jobs:
         if 'label' not in job:
             if 'name' not in job:
                 raise Exception("job has neither a name nor a label")
             job['label'] = '{}-{}'.format(config.kind, job['name'])
         if job['name']:
             del job['name']
-        if 'platform' in job:
-            if 'treeherder' in job:
-                job['treeherder']['platform'] = job['platform']
-            del job['platform']
-
         taskdesc = copy.deepcopy(job)
 
         # fill in some empty defaults to make run implementations easier
         taskdesc.setdefault('attributes', {})
         taskdesc.setdefault('dependencies', {})
         taskdesc.setdefault('routes', [])
         taskdesc.setdefault('scopes', [])
         taskdesc.setdefault('extra', {})
 
         # give the function for job.run.using on this worker implementation a
         # chance to set up the task description.
         configure_taskdesc_for_run(config, job, taskdesc)
         del taskdesc['run']
 
+        if 'platform' in taskdesc:
+            if 'treeherder' in taskdesc:
+                taskdesc['treeherder']['platform'] = taskdesc['platform']
+            del taskdesc['platform']
+
         # yield only the task description, discarding the job description
         yield taskdesc
 
 # A registry of all functions decorated with run_job_using
 registry = {}
 
 
 def run_job_using(worker_implementation, run_using, schema=None):
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -4,16 +4,20 @@
 """
 Common support for various job types.  These functions are all named after the
 worker implementation they operate on, and take the same three parameters, for
 consistency.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+from taskgraph.util.attributes import keymatch
+
+
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
 SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}'
 
 
 def docker_worker_add_workspace_cache(config, job, taskdesc):
     """Add the workspace cache based on the build platform/type and level,
     except on try where workspace caches are not used."""
     if config.params['project'] == 'try':
         return
@@ -52,16 +56,45 @@ def docker_worker_add_gecko_vcs_env_vars
     env.update({
         'GECKO_BASE_REPOSITORY': config.params['base_repository'],
         'GECKO_HEAD_REF': config.params['head_rev'],
         'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
         'GECKO_HEAD_REV': config.params['head_rev'],
     })
 
 
+def docker_worker_add_build_dependency(config, job, taskdesc):
+    """Add build dependency to the task description and installer_url to env."""
+    key = job['platform']
+    build_labels = config.config.get('dependent-build-platforms', {})
+    matches = keymatch(build_labels, key)
+    if not matches:
+        raise Exception("No build platform found for '{}'. "
+                        "Define 'dependent-build-platforms' in the kind config.".format(key))
+
+    if len(matches) > 1:
+        raise Exception("More than one build platform found for '{}'.".format(key))
+
+    label = matches[0]
+    deps = taskdesc.setdefault('dependencies', {})
+    deps.update({'build': label})
+
+    if 'macosx' in label:
+        target = 'target.dmg'
+    elif 'android' in label:
+        target = 'target.apk'
+    else:
+        target = 'target.tar.bz2'
+    build_artifact = 'public/build/{}'.format(target)
+    installer_url = ARTIFACT_URL.format('<build>', build_artifact)
+
+    env = taskdesc['worker'].setdefault('env', {})
+    env.update({'GECKO_INSTALLER_URL': {'task-reference': installer_url}})
+
+
 def docker_worker_support_vcs_checkout(config, job, taskdesc):
     """Update a job/task with parameters to enable a VCS checkout.
 
     The configuration is intended for tasks using "run-task" and its
     VCS checkout behavior.
     """
     level = config.params['level']
 
--- a/taskcluster/taskgraph/transforms/job/mach.py
+++ b/taskcluster/taskgraph/transforms/job/mach.py
@@ -11,16 +11,22 @@ from taskgraph.transforms.job import run
 from taskgraph.transforms.job.run_task import docker_worker_run_task
 from voluptuous import Schema, Required
 
 mach_schema = Schema({
     Required('using'): 'mach',
 
     # The mach command (omitting `./mach`) to run
     Required('mach'): basestring,
+
+    # Whether the job requires a build artifact or not. If True, the task
+    # will depend on a build task and run-task will download and set up the
+    # installer. Build labels are determined by the `dependent-build-platforms`
+    # config in kind.yml.
+    Required('requires-build', default=False): bool,
 })
 
 
 @run_job_using("docker-worker", "mach", schema=mach_schema)
 def docker_worker_mach(config, job, taskdesc):
     run = job['run']
 
     # defer to the run_task implementation
--- a/taskcluster/taskgraph/transforms/job/run_task.py
+++ b/taskcluster/taskgraph/transforms/job/run_task.py
@@ -6,16 +6,17 @@ Support for running jobs that are invoke
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import (
+    docker_worker_add_build_dependency,
     docker_worker_support_vcs_checkout,
 )
 from voluptuous import Schema, Required, Any
 
 run_task_schema = Schema({
     Required('using'): 'run-task',
 
     # if true, add a cache at ~worker/.cache, which is where things like pip
@@ -24,28 +25,37 @@ run_task_schema = Schema({
 
     # if true (the default), perform a checkout in /home/worker/checkouts/gecko
     Required('checkout', default=True): bool,
 
     # The command arguments to pass to the `run-task` script, after the
     # checkout arguments.  If a list, it will be passed directly; otherwise
     # it will be included in a single argument to `bash -cx`.
     Required('command'): Any([basestring], basestring),
+
+    # Whether the job requires a build artifact or not. If True, the task
+    # will depend on a build task and run-task will download and set up the
+    # installer. Build labels are determined by the `dependent-build-platforms`
+    # config in kind.yml.
+    Required('requires-build', default=False): bool,
 })
 
 
 @run_job_using("docker-worker", "run-task", schema=run_task_schema)
 def docker_worker_run_task(config, job, taskdesc):
     run = job['run']
 
     worker = taskdesc['worker'] = copy.deepcopy(job['worker'])
 
     if run['checkout']:
         docker_worker_support_vcs_checkout(config, job, taskdesc)
 
+    if run['requires-build']:
+        docker_worker_add_build_dependency(config, job, taskdesc)
+
     if run.get('cache-dotcache') and int(config.params['level']) > 1:
         worker['caches'].append({
             'type': 'persistent',
             'name': 'level-{level}-{project}-dotcache'.format(**config.params),
             'mount-point': '/home/worker/.cache',
         })
 
     run_command = run['command']