Bug 1532783: [taskgraph] Set worker `os` and `implementation` earlier in job transform; r=dustin
☠☠ backed out by 54ed5eac2abc ☠ ☠
authorTom Prince <mozilla@hocat.ca>
Thu, 07 Mar 2019 15:53:45 +0000
changeset 520970 421bdcc103d3d2f10e1e5f3a8cad736dad789f69
parent 520969 2f913db74eb514ea53dccc1d1f4bfe636a7c8391
child 520971 722b3915da319bd9cab0a9f2a43c18b75585ffec
push id10862
push userffxbld-merge
push dateMon, 11 Mar 2019 13:01:11 +0000
treeherdermozilla-beta@a2e7f5c935da [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1532783
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1532783: [taskgraph] Set worker `os` and `implementation` earlier in job transform; r=dustin This slightly decreases the amount of code that needs to know how to determine this. Differential Revision: https://phabricator.services.mozilla.com/D22446
taskcluster/taskgraph/transforms/job/__init__.py
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -125,16 +125,33 @@ def rewrite_when_to_optimization(config,
 
         # "only when files changed" implies "skip if files have not changed"
         job['optimization'] = {'skip-unless-changed': files_changed}
 
         assert 'when' not in job
         yield job
 
 
+@transforms.add
+def set_implementation(config, jobs):
+    for job in jobs:
+        impl, os = worker_type_implementation(job['worker-type'])
+        if os:
+            job.setdefault('tags', {})['os'] = os
+        if impl:
+            job.setdefault('tags', {})['worker-implementation'] = impl
+        worker = job.setdefault('worker', {})
+        assert 'implementation' not in worker
+        worker['implementation'] = impl
+        if os:
+            worker['os'] = os
+
+        yield job
+
+
 def get_attribute(dict, key, attributes, attribute_name):
     '''Get `attribute_name` from the given `attributes` dict, and if there
     is a corresponding value, set `key` in `dict` to that value.'''
     value = attributes.get(attribute_name)
     if value:
         dict[key] = value
 
 
@@ -201,18 +218,17 @@ def use_fetches(config, jobs):
                     }
                     if dest is not None:
                         fetch['dest'] = dest
                     job_fetches.append(fetch)
 
         env = job.setdefault('worker', {}).setdefault('env', {})
         env['MOZ_FETCHES'] = {'task-reference': json.dumps(job_fetches, sort_keys=True)}
 
-        impl, os = worker_type_implementation(job['worker-type'])
-        if os in ('windows', 'macosx'):
+        if job['worker']['os'] in ('windows', 'macosx'):
             env.setdefault('MOZ_FETCHES_DIR', 'fetches')
         else:
             workdir = job['run'].get('workdir', '/builds/worker')
             env.setdefault('MOZ_FETCHES_DIR', '{}/fetches'.format(workdir))
 
         yield job
 
 
@@ -224,44 +240,33 @@ def make_task_description(config, jobs):
     for job in jobs:
         if 'label' not in job:
             if 'name' not in job:
                 raise Exception("job has neither a name nor a label")
             job['label'] = '{}-{}'.format(config.kind, job['name'])
         if job.get('name'):
             del job['name']
 
-        impl, os = worker_type_implementation(job['worker-type'])
-        if os:
-            job.setdefault('tags', {})['os'] = os
-        if impl:
-            job.setdefault('tags', {})['worker-implementation'] = impl
-        worker = job.setdefault('worker', {})
-        assert 'implementation' not in worker
-        worker['implementation'] = impl
-        if os:
-            worker['os'] = os
-
         # always-optimized tasks never execute, so have no workdir
         if job['run']['using'] != 'always-optimized':
             job['run'].setdefault('workdir', '/builds/worker')
 
         taskdesc = copy.deepcopy(job)
 
         # fill in some empty defaults to make run implementations easier
         taskdesc.setdefault('attributes', {})
         taskdesc.setdefault('dependencies', {})
         taskdesc.setdefault('soft-dependencies', [])
         taskdesc.setdefault('routes', [])
         taskdesc.setdefault('scopes', [])
         taskdesc.setdefault('extra', {})
 
         # give the function for job.run.using on this worker implementation a
         # chance to set up the task description.
-        configure_taskdesc_for_run(config, job, taskdesc, impl)
+        configure_taskdesc_for_run(config, job, taskdesc, job['worker']['implementation'])
         del taskdesc['run']
 
         # yield only the task description, discarding the job description
         yield taskdesc
 
 
 # A registry of all functions decorated with run_job_using
 registry = {}