Bug 1048446 - [taskcluster] Move 'require-build' out of run_task and into source_test, r=dustin
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 29 May 2017 15:34:54 -0400
changeset 411397 b2d113409e2195b5e70b17c52364acdf72dd575c
parent 411396 83081a324efb68dacb018a4ea1ef8dea91796dba
child 411398 c7bcf4a7e382c840f26031f5507b04e7d695b356
push id7391
push usermtabara@mozilla.com
push dateMon, 12 Jun 2017 13:08:53 +0000
treeherdermozilla-beta@2191d7f87e2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1048446
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1048446 - [taskcluster] Move 'require-build' out of run_task and into source_test, r=dustin The 'platform' key was recently moved out of 'job' tasks and into the 'source-test' kind. Since the concept of requiring a build depends on this key, let's move that back to source-test as well. MozReview-Commit-ID: 4bs8G4wN5OH
taskcluster/taskgraph/transforms/job/common.py
taskcluster/taskgraph/transforms/job/mach.py
taskcluster/taskgraph/transforms/job/run_task.py
taskcluster/taskgraph/transforms/source_test.py
--- a/taskcluster/taskgraph/transforms/job/common.py
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -4,20 +4,16 @@
 """
 Common support for various job types.  These functions are all named after the
 worker implementation they operate on, and take the same three parameters, for
 consistency.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-from taskgraph.util.attributes import keymatch
-
-
-ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
 SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}'
 
 
 def docker_worker_add_workspace_cache(config, job, taskdesc):
     """Add the workspace cache based on the build platform/type and level,
     except on try where workspace caches are not used."""
     if config.params['project'] == 'try':
         return
@@ -56,40 +52,16 @@ def docker_worker_add_gecko_vcs_env_vars
     env.update({
         'GECKO_BASE_REPOSITORY': config.params['base_repository'],
         'GECKO_HEAD_REF': config.params['head_rev'],
         'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
         'GECKO_HEAD_REV': config.params['head_rev'],
     })
 
 
-def add_build_dependency(config, job, taskdesc):
-    """Add build dependency to the task description and installer_url to env."""
-    key = job['platform']
-    build_labels = config.config.get('dependent-build-platforms', {})
-    matches = keymatch(build_labels, key)
-    if not matches:
-        raise Exception("No build platform found for '{}'. "
-                        "Define 'dependent-build-platforms' in the kind config.".format(key))
-
-    if len(matches) > 1:
-        raise Exception("More than one build platform found for '{}'.".format(key))
-
-    label = matches[0]['label']
-    target = matches[0]['target-name']
-    deps = taskdesc.setdefault('dependencies', {})
-    deps.update({'build': label})
-
-    build_artifact = 'public/build/{}'.format(target)
-    installer_url = ARTIFACT_URL.format('<build>', build_artifact)
-
-    env = taskdesc['worker'].setdefault('env', {})
-    env.update({'GECKO_INSTALLER_URL': {'task-reference': installer_url}})
-
-
 def support_vcs_checkout(config, job, taskdesc):
     """Update a job/task with parameters to enable a VCS checkout.
 
     The configuration is intended for tasks using "run-task" and its
     VCS checkout behavior.
     """
     level = config.params['level']
 
--- a/taskcluster/taskgraph/transforms/job/mach.py
+++ b/taskcluster/taskgraph/transforms/job/mach.py
@@ -15,22 +15,16 @@ from taskgraph.transforms.job.run_task i
 from taskgraph.util.schema import Schema
 from voluptuous import Required
 
 mach_schema = Schema({
     Required('using'): 'mach',
 
     # The mach command (omitting `./mach`) to run
     Required('mach'): basestring,
-
-    # Whether the job requires a build artifact or not. If True, the task
-    # will depend on a build task and run-task will download and set up the
-    # installer. Build labels are determined by the `dependent-build-platforms`
-    # config in kind.yml.
-    Required('requires-build', default=False): bool,
 })
 
 
 @run_job_using("docker-worker", "mach", schema=mach_schema)
 @run_job_using("native-engine", "mach", schema=mach_schema)
 def docker_worker_mach(config, job, taskdesc):
     run = job['run']
 
--- a/taskcluster/taskgraph/transforms/job/run_task.py
+++ b/taskcluster/taskgraph/transforms/job/run_task.py
@@ -4,53 +4,41 @@
 """
 Support for running jobs that are invoked via the `run-task` script.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.util.schema import Schema
-from taskgraph.transforms.job.common import (
-    add_build_dependency,
-    support_vcs_checkout,
-)
+from taskgraph.transforms.job.common import support_vcs_checkout
 from voluptuous import Required, Any
 
 run_task_schema = Schema({
     Required('using'): 'run-task',
 
     # if true, add a cache at ~worker/.cache, which is where things like pip
     # tend to hide their caches.  This cache is never added for level-1 jobs.
     Required('cache-dotcache', default=False): bool,
 
     # if true (the default), perform a checkout in /home/worker/checkouts/gecko
     Required('checkout', default=True): bool,
 
     # The command arguments to pass to the `run-task` script, after the
     # checkout arguments.  If a list, it will be passed directly; otherwise
     # it will be included in a single argument to `bash -cx`.
     Required('command'): Any([basestring], basestring),
-
-    # Whether the job requires a build artifact or not. If True, the task
-    # will depend on a build task and run-task will download and set up the
-    # installer. Build labels are determined by the `dependent-build-platforms`
-    # config in kind.yml.
-    Required('requires-build', default=False): bool,
 })
 
 
 def common_setup(config, job, taskdesc):
     run = job['run']
     if run['checkout']:
         support_vcs_checkout(config, job, taskdesc)
 
-    if run['requires-build']:
-        add_build_dependency(config, job, taskdesc)
-
 
 @run_job_using("docker-worker", "run-task", schema=run_task_schema)
 def docker_worker_run_task(config, job, taskdesc):
     run = job['run']
     worker = taskdesc['worker'] = job['worker']
     common_setup(config, job, taskdesc)
 
     if run.get('cache-dotcache') and int(config.params['level']) > 1:
--- a/taskcluster/taskgraph/transforms/source_test.py
+++ b/taskcluster/taskgraph/transforms/source_test.py
@@ -8,39 +8,48 @@ treeherder configuration and attributes 
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.transforms.job import job_description_schema
+from taskgraph.util.attributes import keymatch
 from taskgraph.util.schema import (
     validate_schema,
     resolve_keyed_by,
 )
 from voluptuous import (
     Any,
     Extra,
     Required,
     Schema,
 )
 
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
+
 job_description_schema = {str(k): v for k, v in job_description_schema.schema.iteritems()}
 
 source_test_description_schema = Schema({
     # most fields are passed directly through as job fields, and are not
     # repeated here
     Extra: object,
 
     # The platform on which this task runs.  This will be used to set up attributes
     # (for try selection) and treeherder metadata (for display).  If given as a list,
     # the job will be "split" into multiple tasks, one with each platform.
     Required('platform'): Any(basestring, [basestring]),
 
+    # Whether the job requires a build artifact or not. If True, the task will
+    # depend on a build task and the installer url will be saved to the
+    # GECKO_INSTALLER_URL environment variable. Build labels are determined by the
+    # `dependent-build-platforms` config in kind.yml.
+    Required('require-build', default=False): bool,
+
     # These fields can be keyed by "platform", and are otherwise identical to
     # job descriptions.
     Required('worker-type'): Any(
         job_description_schema['worker-type'],
         {'by-platform': {basestring: job_description_schema['worker-type']}},
     ),
     Required('worker'): Any(
         job_description_schema['worker'],
@@ -78,16 +87,42 @@ def expand_platforms(config, jobs):
 
             if 'name' in pjob:
                 pjob['name'] = '{}-{}'.format(pjob['name'], platform)
             else:
                 pjob['label'] = '{}-{}'.format(pjob['label'], platform)
             yield pjob
 
 
+def add_build_dependency(config, job):
+    """
+    Add build dependency to the job and installer_url to env.
+    """
+    key = job['platform']
+    build_labels = config.config.get('dependent-build-platforms', {})
+    matches = keymatch(build_labels, key)
+    if not matches:
+        raise Exception("No build platform found for '{}'. "
+                        "Define 'dependent-build-platforms' in the kind config.".format(key))
+
+    if len(matches) > 1:
+        raise Exception("More than one build platform found for '{}'.".format(key))
+
+    label = matches[0]['label']
+    target = matches[0]['target-name']
+    deps = job.setdefault('dependencies', {})
+    deps.update({'build': label})
+
+    build_artifact = 'public/build/{}'.format(target)
+    installer_url = ARTIFACT_URL.format('<build>', build_artifact)
+
+    env = job['worker'].setdefault('env', {})
+    env.update({'GECKO_INSTALLER_URL': {'task-reference': installer_url}})
+
+
 @transforms.add
 def handle_platform(config, jobs):
     """
     Handle the 'platform' property, setting up treeherder context as well as
     try-related attributes.
     """
     fields = [
         'worker-type',
@@ -98,10 +133,13 @@ def handle_platform(config, jobs):
         platform = job['platform']
 
         for field in fields:
             resolve_keyed_by(job, field, item_name=job['name'])
 
         if 'treeherder' in job:
             job['treeherder']['platform'] = platform
 
+        if job.pop('require-build'):
+            add_build_dependency(config, job)
+
         del job['platform']
         yield job