Bug 1379163: make parameterization functions into utilities; r=bstack
authorDustin J. Mitchell <dustin@mozilla.com>
Thu, 20 Jul 2017 19:24:50 +0000
changeset 421762 2e143d567065e0ae90fa11268225b6b5572bdedd
parent 421761 963b59bd61a365410ab14867e1d2994a7b2f94c7
child 421763 aff22b63df7a936f2f3f94ac86ad78d43123f12f
push id1517
push userjlorenzo@mozilla.com
push dateThu, 14 Sep 2017 16:50:54 +0000
treeherdermozilla-release@3b41fd564418 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbstack
bugs1379163
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1379163: make parameterization functions into utilities; r=bstack These both have a similar form, recursing over a nested JSON structure, and are useful outside of the modules in which they are defined. MozReview-Commit-ID: 1bsRtlaQol7
taskcluster/taskgraph/create.py
taskcluster/taskgraph/optimize.py
taskcluster/taskgraph/test/python.ini
taskcluster/taskgraph/test/test_util_parameterization.py
taskcluster/taskgraph/util/parameterization.py
--- a/taskcluster/taskgraph/create.py
+++ b/taskcluster/taskgraph/create.py
@@ -7,20 +7,18 @@ from __future__ import absolute_import, 
 import concurrent.futures as futures
 import requests
 import requests.adapters
 import json
 import os
 import logging
 
 from slugid import nice as slugid
-from taskgraph.util.time import (
-    current_json_time,
-    json_time_from_now
-)
+from taskgraph.util.parameterization import resolve_timestamps
+from taskgraph.util.time import current_json_time
 
 logger = logging.getLogger(__name__)
 
 # the maximum number of parallel createTask calls to make
 CONCURRENCY = 50
 
 
 def create_tasks(taskgraph, label_to_taskid, params):
@@ -101,22 +99,8 @@ def create_task(session, task_id, label,
     res = session.put('http://taskcluster/queue/v1/task/{}'.format(task_id),
                       data=json.dumps(task_def))
     if res.status_code != 200:
         try:
             logger.error(res.json()['message'])
         except:
             logger.error(res.text)
         res.raise_for_status()
-
-
-def resolve_timestamps(now, task_def):
-    def recurse(val):
-        if isinstance(val, list):
-            return [recurse(v) for v in val]
-        elif isinstance(val, dict):
-            if val.keys() == ['relative-datestamp']:
-                return json_time_from_now(val['relative-datestamp'], now)
-            else:
-                return {k: recurse(v) for k, v in val.iteritems()}
-        else:
-            return val
-    return recurse(task_def)
--- a/taskcluster/taskgraph/optimize.py
+++ b/taskcluster/taskgraph/optimize.py
@@ -1,28 +1,27 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import logging
-import re
 import os
 import requests
 
 from .graph import Graph
 from . import files_changed
 from .taskgraph import TaskGraph
 from .util.seta import is_low_value_task
 from .util.taskcluster import find_task_id
+from .util.parameterization import resolve_task_references
 from slugid import nice as slugid
 
 logger = logging.getLogger(__name__)
-TASK_REFERENCE_PATTERN = re.compile('<([^>]+)>')
 
 _optimizations = {}
 
 
 def optimize_task_graph(target_task_graph, params, do_not_optimize, existing_tasks=None):
     """
     Perform task optimization, without optimizing tasks named in
     do_not_optimize.
@@ -39,40 +38,16 @@ def optimize_task_graph(target_task_grap
                         params=params,
                         do_not_optimize=do_not_optimize,
                         named_links_dict=named_links_dict,
                         label_to_taskid=label_to_taskid,
                         existing_tasks=existing_tasks)
     return get_subgraph(target_task_graph, named_links_dict, label_to_taskid), label_to_taskid
 
 
-def resolve_task_references(label, task_def, taskid_for_edge_name):
-    def repl(match):
-        key = match.group(1)
-        try:
-            return taskid_for_edge_name[key]
-        except KeyError:
-            # handle escaping '<'
-            if key == '<':
-                return key
-            raise KeyError("task '{}' has no dependency named '{}'".format(label, key))
-
-    def recurse(val):
-        if isinstance(val, list):
-            return [recurse(v) for v in val]
-        elif isinstance(val, dict):
-            if val.keys() == ['task-reference']:
-                return TASK_REFERENCE_PATTERN.sub(repl, val['task-reference'])
-            else:
-                return {k: recurse(v) for k, v in val.iteritems()}
-        else:
-            return val
-    return recurse(task_def)
-
-
 def optimize_task(task, params):
     """
     Optimize a single task by running its optimizations in order until one
     succeeds.
     """
     for opt in task.optimizations:
         opt_type, args = opt[0], opt[1:]
         opt_fn = _optimizations[opt_type]
--- a/taskcluster/taskgraph/test/python.ini
+++ b/taskcluster/taskgraph/test/python.ini
@@ -11,14 +11,15 @@ subsuite = taskgraph
 [test_optimize.py]
 [test_parameters.py]
 [test_target_tasks.py]
 [test_taskgraph.py]
 [test_transforms_base.py]
 [test_try_option_syntax.py]
 [test_util_attributes.py]
 [test_util_docker.py]
+[test_util_parameterization.py]
 [test_util_python_path.py]
 [test_util_schema.py]
 [test_util_templates.py]
 [test_util_time.py]
 [test_util_treeherder.py]
 [test_util_yaml.py]
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_util_parameterization.py
@@ -0,0 +1,70 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+import datetime
+
+from mozunit import main
+from taskgraph.util.parameterization import (
+    resolve_timestamps,
+    resolve_task_references,
+)
+
+
+class TestTimestamps(unittest.TestCase):
+
+    def test_no_change(self):
+        now = datetime.datetime(2018, 01, 01)
+        input = {
+            "key": "value",
+            "numeric": 10,
+            "list": ["a", True, False, None],
+        }
+        self.assertEqual(resolve_timestamps(now, input), input)
+
+    def test_buried_replacement(self):
+        now = datetime.datetime(2018, 01, 01)
+        input = {"key": [{"key2": [{'relative-datestamp': '1 day'}]}]}
+        self.assertEqual(resolve_timestamps(now, input),
+                         {"key": [{"key2": ['2018-01-02T00:00:00Z']}]})
+
+    def test_appears_with_other_keys(self):
+        now = datetime.datetime(2018, 01, 01)
+        input = [{'relative-datestamp': '1 day', 'another-key': True}]
+        self.assertEqual(resolve_timestamps(now, input),
+                         [{'relative-datestamp': '1 day', 'another-key': True}])
+
+
+class TestTaskRefs(unittest.TestCase):
+
+    def test_no_change(self):
+        input = {"key": "value", "numeric": 10, "list": ["a", True, False, None]}
+        self.assertEqual(resolve_task_references('lable', input, {}), input)
+
+    def test_buried_replacement(self):
+        input = {"key": [{"key2": [{'task-reference': 'taskid=<toolchain>'}]}]}
+        self.assertEqual(resolve_task_references('lable', input, {'toolchain': 'abcd'}),
+                         {u'key': [{u'key2': [u'taskid=abcd']}]})
+
+    def test_appears_with_other_keys(self):
+        input = [{'task-reference': '<toolchain>', 'another-key': True}]
+        self.assertEqual(resolve_task_references('lable', input, {'toolchain': 'abcd'}),
+                         [{'task-reference': '<toolchain>', 'another-key': True}])
+
+    def test_multiple_subs(self):
+        input = [{'task-reference': 'toolchain=<toolchain>, build=<build>'}]
+        self.assertEqual(
+            resolve_task_references('lable', input, {'toolchain': 'abcd', 'build': 'def'}),
+            ['toolchain=abcd, build=def'])
+
+    def test_escaped(self):
+        input = [{'task-reference': '<<><toolchain>>'}]
+        self.assertEqual(resolve_task_references('lable', input, {'toolchain': 'abcd'}),
+                         ['<abcd>'])
+
+
+if __name__ == '__main__':
+    main()
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/util/parameterization.py
@@ -0,0 +1,48 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import re
+
+from taskgraph.util.time import json_time_from_now
+
+TASK_REFERENCE_PATTERN = re.compile('<([^>]+)>')
+
+
+def _recurse(val, param_name, param_fn):
+    param_keys = [param_name]
+
+    def recurse(val):
+        if isinstance(val, list):
+            return [recurse(v) for v in val]
+        elif isinstance(val, dict):
+            if val.keys() == param_keys:
+                return param_fn(val[param_name])
+            else:
+                return {k: recurse(v) for k, v in val.iteritems()}
+        else:
+            return val
+    return recurse(val)
+
+
+def resolve_timestamps(now, task_def):
+    """Resolve all instances of `{'relative-datestamp': '..'}` in the given task definition"""
+    return _recurse(task_def, 'relative-datestamp', lambda v: json_time_from_now(v, now))
+
+
+def resolve_task_references(label, task_def, dependencies):
+    """Resolve all instances of `{'task-reference': '..<..>..'}` in the given task
+    definition, using the given dependencies"""
+    def repl(match):
+        key = match.group(1)
+        try:
+            return dependencies[key]
+        except KeyError:
+            # handle escaping '<'
+            if key == '<':
+                return key
+            raise KeyError("task '{}' has no dependency named '{}'".format(label, key))
+
+    return _recurse(task_def, 'task-reference', lambda v: TASK_REFERENCE_PATTERN.sub(repl, v))