Bug 1605275: [taskgraph] Provide a function to get all tasks from a previous set of graphs; r=Callek
authorTom Prince <mozilla@hocat.ca>
Tue, 07 Jan 2020 18:08:42 +0000
changeset 509148 ca6ad3058d2089261bdda75d6f39d4b0bbd412f3
parent 509147 1d063382440d07001ee642d4fbbb6a2e1d03d192
child 509149 89729559c92fa50d1f00a880e6c6824431c7cb22
push id36993
push userdluca@mozilla.com
push dateWed, 08 Jan 2020 09:41:58 +0000
treeherdermozilla-central@12fb5e522dd3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersCallek
bugs1605275
milestone74.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1605275: [taskgraph] Provide a function to get all tasks from a previous set of graphs; r=Callek Differential Revision: https://phabricator.services.mozilla.com/D58450
taskcluster/taskgraph/util/taskgraph.py
tools/tryselect/selectors/scriptworker.py
--- a/taskcluster/taskgraph/util/taskgraph.py
+++ b/taskcluster/taskgraph/util/taskgraph.py
@@ -20,21 +20,33 @@ def find_decision_task(parameters, graph
     """Given the parameters for this action, find the taskId of the decision
     task"""
     return find_task_id('{}.v2.{}.pushlog-id.{}.decision'.format(
         graph_config['trust-domain'],
         parameters['project'],
         parameters['pushlog_id']))
 
 
-def find_existing_tasks_from_previous_kinds(full_task_graph, previous_graph_ids,
-                                            rebuild_kinds):
+def find_existing_tasks(previous_graph_ids):
+    existing_tasks = {}
+    for previous_graph_id in previous_graph_ids:
+        label_to_taskid = get_artifact(previous_graph_id, "public/label-to-taskid.json")
+        existing_tasks.update(label_to_taskid)
+    return existing_tasks
+
+
+def find_existing_tasks_from_previous_kinds(
+    full_task_graph, previous_graph_ids, rebuild_kinds
+):
     """Given a list of previous decision/action taskIds and kinds to ignore
     from the previous graphs, return a dictionary of labels-to-taskids to use
     as ``existing_tasks`` in the optimization step."""
-    existing_tasks = {}
-    for previous_graph_id in previous_graph_ids:
-        label_to_taskid = get_artifact(previous_graph_id, "public/label-to-taskid.json")
-        kind_labels = set(t.label for t in full_task_graph.tasks.itervalues()
-                          if t.attributes['kind'] not in rebuild_kinds)
-        for label in set(label_to_taskid.keys()).intersection(kind_labels):
-            existing_tasks[label] = label_to_taskid[label]
-    return existing_tasks
+    existing_tasks = find_existing_tasks(previous_graph_ids)
+    kind_labels = {
+        t.label
+        for t in full_task_graph.tasks.itervalues()
+        if t.attributes["kind"] not in rebuild_kinds
+    }
+    return {
+        label: taskid
+        for (label, taskid) in existing_tasks.items()
+        if label in kind_labels
+    }
--- a/tools/tryselect/selectors/scriptworker.py
+++ b/tools/tryselect/selectors/scriptworker.py
@@ -3,21 +3,20 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import sys
 
 import requests
 
-from ..tasks import generate_tasks
 from ..cli import BaseTryParser
 from ..push import push_to_try
 
-from taskgraph.util.taskgraph import find_existing_tasks_from_previous_kinds
+from taskgraph.util.taskgraph import find_existing_tasks
 from taskgraph.util.taskcluster import get_artifact, get_session
 from taskgraph.parameters import Parameters
 
 TASK_TYPES = {
     "linux-signing": [
         "build-signing-linux-shippable/opt",
         "build-signing-linux64-shippable/opt",
         "build-signing-win64-shippable/opt",
@@ -71,24 +70,16 @@ def get_releases(branch):
     )
     if response.status_code == 403:
         print("Cannont connect to ship-it. Are you on the VPN and have shipit access?")
         sys.exit(1)
     response.raise_for_status()
     return response.json()
 
 
-def find_existing_tasks(graph_id):
-
-    full_task_graph = generate_tasks(full=True)
-    return find_existing_tasks_from_previous_kinds(
-        full_task_graph, [graph_id], rebuild_kinds=[]
-    )
-
-
 def get_ship_phase_graph(release):
     for phase in release["phases"]:
         if phase["name"] in ("ship_firefox",):
             return phase["actionTaskId"]
     raise Exception("No ship phase.")
 
 
 def print_available_task_types():
@@ -115,17 +106,17 @@ def run(
     closed_tree=False,
 ):
     if task_type == "list":
         print_available_task_types()
         sys.exit(0)
 
     release = get_releases(RELEASE_TO_BRANCH[release_type])[-1]
     ship_graph = get_ship_phase_graph(release)
-    existing_tasks = find_existing_tasks(ship_graph)
+    existing_tasks = find_existing_tasks([ship_graph])
 
     ship_parameters = Parameters(
         strict=False, **get_artifact(ship_graph, "public/parameters.yml")
     )
 
     # Copy L10n configuration from the commit the release we are using was
     # based on. This *should* ensure that the chunking of L10n tasks is the
     # same between graphs.