Bug 1397847: Add supported API to get tasks of a given kind from a taskgraph; r=dustin
authorTom Prince <mozilla@hocat.ca>
Wed, 28 Nov 2018 17:55:35 +0000
changeset 507766 393f02fb1ad734a1f37a2c33ac29050a3130cb23
parent 507765 3b37632be987ce0ae9df8354eaa46ce91c73c18a
child 507767 da3635436bf11b00795861958e251249a8fe005c
push id1905
push userffxbld-merge
push dateMon, 21 Jan 2019 12:33:13 +0000
treeherdermozilla-release@c2fca1944d8c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1397847
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1397847: Add supported API to get tasks of a given kind from a taskgraph; r=dustin `mach artifact toolchain` gets task definitions from taskgraph, to get the index path to find the artifacts at. Now that these index paths depend on the digests of fetch tasks, those kinds need to be loaded as well. This adds a supported API to get task definitions for a given kind, which loads all the kind dependencies. Differential Revision: https://phabricator.services.mozilla.com/D12161
python/mozbuild/mozbuild/mach_commands.py
taskcluster/mach_commands.py
taskcluster/taskgraph/generator.py
taskcluster/taskgraph/transforms/job/__init__.py
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1422,33 +1422,23 @@ class PackageFrontend(MachCommandBase):
         if from_build:
             if 'MOZ_AUTOMATION' in os.environ:
                 self.log(logging.ERROR, 'artifact', {},
                          'Do not use --from-build in automation; all dependencies '
                          'should be determined in the decision task.')
                 return 1
             from taskgraph.optimize import IndexSearch
             from taskgraph.parameters import Parameters
+            from taskgraph.generator import load_tasks_for_kind
             params = Parameters(
                 level=os.environ.get('MOZ_SCM_LEVEL', '3'),
                 strict=False,
-                ignore_fetches=True,
             )
 
-            # TODO: move to the taskcluster package
-            def tasks(kind_name):
-                root_path = mozpath.join(self.topsrcdir, 'taskcluster', 'ci')
-                graph_config = load_graph_config(root_path)
-                tasks = Kind.load(root_path, graph_config, kind_name).load_tasks(params, {})
-                return {
-                    task.task['metadata']['name']: task
-                    for task in tasks
-                }
-
-            toolchains = tasks('toolchain')
+            toolchains = load_tasks_for_kind(params, 'toolchain')
 
             aliases = {}
             for t in toolchains.values():
                 alias = t.attributes.get('toolchain-alias')
                 if alias:
                     aliases['toolchain-{}'.format(alias)] = \
                         t.task['metadata']['name']
 
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -47,18 +47,21 @@ class ShowTaskGraphSubCommand(SubCommand
                                  "`taskcluster/docs/parameters.rst`)`"),
             CommandArgument('--no-optimize', dest="optimize", action="store_false",
                             default="true",
                             help="do not remove tasks from the graph that are found in the "
                             "index (a.k.a. optimize the graph)"),
             CommandArgument('--tasks-regex', '--tasks', default=None,
                             help="only return tasks with labels matching this regular "
                             "expression."),
+            CommandArgument('--target-kind', default=None,
+                            help="only return tasks that are of the given kind, "
+                                 "or their dependencies."),
             CommandArgument('-F', '--fast', dest='fast', default=False, action='store_true',
-                            help="enable fast task generation for local debugging.")
+                            help="enable fast task generation for local debugging."),
 
         ]
         for arg in args:
             after = arg(after)
         return after
 
 
 @CommandProvider
@@ -326,30 +329,31 @@ class MachCommands(MachCommandBase):
                 write_interval=old.formatter.write_interval,
                 write_times=old.formatter.write_times)
 
         # all of the taskgraph logging is unstructured logging
         self.log_manager.enable_unstructured()
 
     def show_taskgraph(self, graph_attr, options):
         import taskgraph.parameters
-        import taskgraph.target_tasks
         import taskgraph.generator
         import taskgraph
         if options['fast']:
             taskgraph.fast = True
 
         try:
             self.setup_logging(quiet=options['quiet'], verbose=options['verbose'])
             parameters = taskgraph.parameters.load_parameters_file(options['parameters'])
             parameters.check()
 
             tgg = taskgraph.generator.TaskGraphGenerator(
                 root_dir=options.get('root'),
-                parameters=parameters)
+                parameters=parameters,
+                target_kind=options.get('target_kind'),
+            )
 
             tg = getattr(tgg, graph_attr)
 
             show_method = getattr(self, 'show_taskgraph_' + (options['format'] or 'labels'))
             tg = self.get_filtered_taskgraph(tg, options["tasks_regex"])
             show_method(tg)
         except Exception:
             traceback.print_exc()
@@ -385,17 +389,16 @@ class MachCommands(MachCommandBase):
                 for depname, dep in named_links_dict[key].iteritems():
                     if regexprogram.match(dep):
                         filterededges.add((key, dep, depname))
         filtered_taskgraph = TaskGraph(filteredtasks, Graph(set(filteredtasks), filterededges))
         return filtered_taskgraph
 
     def show_actions(self, options):
         import taskgraph.parameters
-        import taskgraph.target_tasks
         import taskgraph.generator
         import taskgraph
         import taskgraph.actions
 
         try:
             self.setup_logging(quiet=options['quiet'], verbose=options['verbose'])
             parameters = taskgraph.parameters.load_parameters_file(options['parameters'])
             parameters.check()
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -100,27 +100,28 @@ class TaskGraphGenerator(object):
     various phases of generation, is available via properties.  This encourages
     the provision of all generation inputs at instance construction time.
     """
 
     # Task-graph generation is implemented as a Python generator that yields
     # each "phase" of generation.  This allows some mach subcommands to short-
     # circuit generation of the entire graph by never completing the generator.
 
-    def __init__(self, root_dir, parameters):
+    def __init__(self, root_dir, parameters, target_kind=None):
         """
         @param root_dir: root directory, with subdirectories for each kind
         @param paramaters: parameters for this task-graph generation, or callable
             taking a `GraphConfig` and returning parameters
         @type parameters: Union[Parameters, Callable[[GraphConfig], Parameters]]
         """
         if root_dir is None:
             root_dir = 'taskcluster/ci'
         self.root_dir = root_dir
         self._parameters = parameters
+        self._target_kind = target_kind
 
         # start the generator
         self._run = self._run()
         self._run_results = {}
 
     @property
     def parameters(self):
         """
@@ -242,16 +243,22 @@ class TaskGraphGenerator(object):
         self.verify_kinds(kinds)
 
         edges = set()
         for kind in kinds.itervalues():
             for dep in kind.config.get('kind-dependencies', []):
                 edges.add((kind.name, dep, 'kind-dependency'))
         kind_graph = Graph(set(kinds), edges)
 
+        if self._target_kind:
+            logger.info(
+                "Limiting kinds to {target_kind} and dependencies".format(
+                    target_kind=self._target_kind))
+            kind_graph = kind_graph.transitive_closure({self._target_kind})
+
         logger.info("Generating full task set")
         all_tasks = {}
         for kind_name in kind_graph.visit_postorder():
             logger.debug("Loading tasks for kind {}".format(kind_name))
             kind = kinds[kind_name]
             try:
                 new_tasks = kind.load_tasks(parameters, list(all_tasks.values()))
             except Exception:
@@ -368,8 +375,22 @@ class TaskGraphGenerator(object):
 
     def verify_run_using(self):
         from .transforms.job import registry
         verify_docs(
             filename="transforms.rst",
             identifiers=registry.keys(),
             appearing_as="inline-literal"
          )
+
+
+def load_tasks_for_kind(parameters, kind):
+    """
+    Get all the tasks of a given kind.
+
+    This function is designed to be called from outside of taskgraph.
+    """
+    tgg = TaskGraphGenerator(root_dir=None, parameters=parameters, target_kind=kind)
+    return {
+        task.task['metadata']['name']: task
+        for task in tgg.full_task_set
+        if task.kind == kind
+    }
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -152,21 +152,16 @@ def use_fetches(config, jobs):
             )
 
     for job in jobs:
         fetches = job.pop('fetches', None)
         if not fetches:
             yield job
             continue
 
-        # Hack added for `mach artifact toolchain` to support reading toolchain
-        # kinds in isolation.
-        if 'fetch' in fetches and config.params.get('ignore_fetches'):
-            fetches['fetch'][:] = []
-
         job_fetches = []
         name = job.get('name', job.get('label'))
         dependencies = job.setdefault('dependencies', {})
         prefix = get_artifact_prefix(job)
         for kind, artifacts in fetches.items():
             if kind in ('fetch', 'toolchain'):
                 for fetch_name in artifacts:
                     label = '{kind}-{name}'.format(kind=kind, name=fetch_name)