Bug 1532710: [taskgraph] Provide a function version of `resolve_keyed_by` that doesn't mutate; r=dustin
authorTom Prince <mozilla@hocat.ca>
Thu, 07 Mar 2019 04:20:33 +0000
changeset 520697 281171d69172b03ff560d9a1702aeac9c5f35c66
parent 520696 395886e7b66bffae4928f3a5e92945b77dc3b3a0
child 520698 0db48ebeac0795acee5b84c8f133d77a6b74f4fb
push id10862
push userffxbld-merge
push dateMon, 11 Mar 2019 13:01:11 +0000
treeherdermozilla-beta@a2e7f5c935da [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1532710
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1532710: [taskgraph] Provide a function version of `resolve_keyed_by` that doesn't mutate; r=dustin I'd like to use the same format for config values, that get evaluated in different contexts, so don't to mutate the config for that. Differential Revision: https://phabricator.services.mozilla.com/D22126
taskcluster/taskgraph/util/keyed_by.py
taskcluster/taskgraph/util/schema.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/util/keyed_by.py
@@ -0,0 +1,74 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from .attributes import keymatch
+
+
+def evaluate_keyed_by(value, item_name, attributes):
+    """
+    For values which can either accept a literal value, or be keyed by some
+    attributes, perform that lookup and return the result.
+
+    For example, given item::
+
+        by-test-platform:
+            macosx-10.11/debug: 13
+            win.*: 6
+            default: 12
+
+    a call to `evaluate_keyed_by(item, 'thing-name', {'test-platform': 'linux96')`
+    would return `12`.
+
+    The `item_name` parameter is used to generate useful error messages.
+    Items can be nested as deeply as desired::
+
+        by-test-platform:
+            win.*:
+                by-project:
+                    ash: ..
+                    cedar: ..
+            linux: 13
+            default: 12
+    """
+    while True:
+        if not isinstance(value, dict) or len(value) != 1 or not value.keys()[0].startswith('by-'):
+            return value
+
+        keyed_by = value.keys()[0][3:]  # strip off 'by-' prefix
+        key = attributes.get(keyed_by)
+        alternatives = value.values()[0]
+
+        if len(alternatives) == 1 and 'default' in alternatives:
+            # Error out when only 'default' is specified as only alternatives,
+            # because we don't need to by-{keyed_by} there.
+            raise Exception(
+                "Keyed-by '{}' unnecessary with only value 'default' "
+                "found, when determining item {}".format(
+                    keyed_by, item_name))
+
+        if key is None:
+            if 'default' in alternatives:
+                value = alternatives['default']
+                continue
+            else:
+                raise Exception(
+                    "No attribute {} and no value for 'default' found "
+                    "while determining item {}".format(
+                        keyed_by, item_name))
+
+        matches = keymatch(alternatives, key)
+        if len(matches) > 1:
+            raise Exception(
+                "Multiple matching values for {} {!r} found while "
+                "determining item {}".format(
+                    keyed_by, key, item_name))
+        elif matches:
+            value = matches[0]
+            continue
+
+        raise Exception(
+            "No {} matching {!r} nor 'default' found while determining item {}".format(
+                keyed_by, key, item_name))
--- a/taskcluster/taskgraph/util/schema.py
+++ b/taskcluster/taskgraph/util/schema.py
@@ -7,17 +7,18 @@ from __future__ import absolute_import, 
 import re
 import pprint
 import collections
 import voluptuous
 
 import taskgraph
 
 from mozbuild import schedules
-from .attributes import keymatch
+
+from .keyed_by import evaluate_keyed_by
 
 
 def validate_schema(schema, obj, msg_prefix):
     """
     Validate that object satisfies schema.  If not, generate a useful exception
     beginning with msg_prefix.
     """
     if taskgraph.fast:
@@ -74,16 +75,17 @@ def resolve_keyed_by(item, field, item_n
                     macosx-10.11/debug: 13
                     win.*: 6
                     default: 12
 
     a call to `resolve_keyed_by(item, 'job.chunks', item['thing-name'])`
     would mutate item in-place to::
 
         job:
+            test-platform: linux128
             chunks: 12
 
     The `item_name` parameter is used to generate useful error messages.
 
     If extra_values are supplied, they represent additional values available
     for reference from by-<field>.
 
     Items can be nested as deeply as the schema will allow::
@@ -104,56 +106,24 @@ def resolve_keyed_by(item, field, item_n
         if f not in container:
             return item
         container = container[f]
         if not isinstance(container, dict):
             return item
 
     if subfield not in container:
         return item
-    value = container[subfield]
-    while True:
-        if not isinstance(value, dict) or len(value) != 1 or not value.keys()[0].startswith('by-'):
-            return item
-
-        keyed_by = value.keys()[0][3:]  # strip off 'by-' prefix
-        key = extra_values[keyed_by] if keyed_by in extra_values else item.get(keyed_by)
-        alternatives = value.values()[0]
-
-        if len(alternatives) == 1 and 'default' in alternatives:
-            # Error out when only 'default' is specified as only alternatives,
-            # because we don't need to by-{keyed_by} there.
-            raise Exception(
-                "Keyed-by '{}' unnecessary with only value 'default' "
-                "found, when determining item '{}' in '{}'".format(
-                    keyed_by, field, item_name))
 
-        if key is None:
-            if 'default' in alternatives:
-                value = container[subfield] = alternatives['default']
-                continue
-            else:
-                raise Exception(
-                    "No attribute {} and no value for 'default' found "
-                    "while determining item {} in {}".format(
-                        keyed_by, field, item_name))
+    container[subfield] = evaluate_keyed_by(
+        value=container[subfield],
+        item_name="`{}` in `{}`".format(field, item_name),
+        attributes=dict(item, **extra_values),
+    )
 
-        matches = keymatch(alternatives, key)
-        if len(matches) > 1:
-            raise Exception(
-                "Multiple matching values for {} {!r} found while "
-                "determining item {} in {}".format(
-                    keyed_by, key, field, item_name))
-        elif matches:
-            value = container[subfield] = matches[0]
-            continue
-
-        raise Exception(
-            "No {} matching {!r} nor 'default' found while determining item {} in {}".format(
-                keyed_by, key, field, item_name))
+    return item
 
 
 # Schemas for YAML files should use dashed identifiers by default.  If there are
 # components of the schema for which there is a good reason to use another format,
 # they can be whitelisted here.
 WHITELISTED_SCHEMA_IDENTIFIERS = [
     # upstream-artifacts are handed directly to scriptWorker, which expects interCaps
     lambda path: "[u'upstream-artifacts']" in path,