Bug 1545862 - refactor to generalize worker-type aliasing r=tomprince
authorDustin J. Mitchell <dustin@mozilla.com>
Wed, 15 May 2019 17:40:02 +0000
changeset 532793 77efc33e0c39eebdfe1b702781e03e719c69922d
parent 532792 8b7a74cee6c4a11fb0dd980409288b4b3c299e6c
child 532794 23d241c9666593bd287ab3333a0227963bd28bcf
push id11272
push userapavel@mozilla.com
push dateThu, 16 May 2019 15:28:22 +0000
treeherdermozilla-beta@2265bfc5920d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstomprince
bugs1545862
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1545862 - refactor to generalize worker-type aliasing r=tomprince Differential Revision: https://phabricator.services.mozilla.com/D29129
taskcluster/taskgraph/util/workertypes.py
--- a/taskcluster/taskgraph/util/workertypes.py
+++ b/taskcluster/taskgraph/util/workertypes.py
@@ -2,16 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from mozbuild.util import memoize
 
 from .keyed_by import evaluate_keyed_by
+from .attributes import keymatch
 
 WORKER_TYPES = {
     'gce/gecko-1-b-linux': ('docker-worker', 'linux'),
     'releng-hardware/gecko-1-b-win2012-gamma': ('generic-worker', 'windows'),
     'gce/gecko-2-b-linux': ('docker-worker', 'linux'),
     'releng-hardware/gecko-2-b-win2012-gamma': ('generic-worker', 'windows'),
     'gce/gecko-3-b-linux': ('docker-worker', 'linux'),
     'releng-hardware/gecko-3-b-win2012-gamma': ('generic-worker', 'windows'),
@@ -26,45 +27,63 @@ WORKER_TYPES = {
     "scriptworker-prov-v1/shipit-dev": ('shipit', None),
     "scriptworker-prov-v1/treescript-v1": ('treescript', None),
     'terraform-packet/gecko-t-linux': ('docker-worker', 'linux'),
     'releng-hardware/gecko-t-osx-1014': ('generic-worker', 'macosx'),
 }
 
 
 @memoize
+def _get(graph_config, alias, level):
+    """Get the configuration for this worker_type alias: {provisioner,
+    worker-type, implementation, os}"""
+    level = str(level)
+
+    # handle the legacy (non-alias) format
+    if '/' in alias:
+        alias = alias.format(level=level)
+        provisioner, worker_type = alias.split("/", 1)
+        try:
+            implementation, os = WORKER_TYPES[alias]
+            return {
+                'provisioner': provisioner,
+                'worker-type': worker_type,
+                'implementation': implementation,
+                'os': os,
+            }
+        except KeyError:
+            return {
+                'provisioner': provisioner,
+                'worker-type': worker_type,
+            }
+
+    matches = keymatch(graph_config['workers']['aliases'], alias)
+    if len(matches) > 1:
+        raise KeyError("Multiple matches for worker-type alias " + alias)
+    elif not matches:
+        raise KeyError("No matches for worker-type alias " + alias)
+    worker_config = matches[0].copy()
+
+    worker_config['worker-type'] = evaluate_keyed_by(
+        worker_config['worker-type'],
+        "worker-type alias {} field worker-type".format(alias),
+        {"level": level}).format(level=level, alias=alias)
+
+    return worker_config
+
+
+@memoize
 def worker_type_implementation(graph_config, worker_type):
     """Get the worker implementation and OS for the given workerType, where the
     OS represents the host system, not the target OS, in the case of
     cross-compiles."""
-    if '/' in worker_type:
-        worker_type = worker_type.replace('{level}', '1')
-        return WORKER_TYPES[worker_type]
-    else:
-        worker_config = evaluate_keyed_by(
-            {"by-worker-type": graph_config["workers"]["aliases"]},
-            "worker-types.yml",
-            {'worker-type': worker_type},
-        )
-        return worker_config['implementation'], worker_config['os']
+    worker_config = _get(graph_config, worker_type, '1')
+    return worker_config['implementation'], worker_config['os']
 
 
 @memoize
 def get_worker_type(graph_config, worker_type, level):
     """
-    Get the worker type based, evaluating aliases from the graph config.
+    Get the worker type provisioner and worker-type, optionally evaluating
+    aliases from the graph config.
     """
-    level = str(level)
-    if '/' in worker_type:
-        worker_type = worker_type.format(level=level)
-        return worker_type.split("/", 1)
-    else:
-        worker_config = evaluate_keyed_by(
-            {"by-worker-type": graph_config["workers"]["aliases"]},
-            "worker-types.yml",
-            {"worker-type": worker_type},
-        )
-        worker_type = evaluate_keyed_by(
-            worker_config["worker-type"],
-            worker_type,
-            {"level": level},
-        ).format(level=level, alias=worker_type)
-        return worker_config["provisioner"], worker_type
+    worker_config = _get(graph_config, worker_type, level)
+    return worker_config["provisioner"], worker_config['worker-type']