Bug 1415199: Move some logic for loading taskgraph kinds from mozbuild.mach_bootstrap to taskgraph. r=dustin
authorTom Prince <mozilla@hocat.ca>
Tue, 07 Nov 2017 10:08:49 -0700
changeset 443872 e605cf1f7c60c4dd0c072d17e628a9b6297a15f7
parent 443871 6ca404d46fc283ec150c7ecfc7197415910d3fb2
child 443873 3681cc015ec7894e50304f2a1239b98b7c886a9c
push id1618
push userCallek@gmail.com
push dateThu, 11 Jan 2018 17:45:48 +0000
treeherdermozilla-release@882ca853e05a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1415199
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1415199: Move some logic for loading taskgraph kinds from mozbuild.mach_bootstrap to taskgraph. r=dustin This fixes the fallout from the mach command not being updated for 69c60c8fc528d8d038e22a2ae62f5ff7b1131231. MozReview-Commit-ID: 7WZ8wW4gnCY
python/mozbuild/mozbuild/mach_commands.py
taskcluster/taskgraph/generator.py
taskcluster/taskgraph/test/test_generator.py
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1779,17 +1779,17 @@ class PackageFrontend(MachCommandBase):
             open_manifest,
             unpack_file,
         )
         from requests.adapters import HTTPAdapter
         import redo
         import requests
         import shutil
 
-        from taskgraph.generator import Kind
+        from taskgraph.generator import load_graph_config, Kind
         from taskgraph.util.taskcluster import (
             get_artifact_url,
             list_artifacts,
         )
         import yaml
 
         self._set_log_level(verbose)
         # Normally, we'd use self.log_manager.enable_unstructured(),
@@ -1900,25 +1900,24 @@ class PackageFrontend(MachCommandBase):
                 'head_rev': '',
                 'moz_build_date': '',
                 'build_date': 0,
                 'pushlog_id': 0,
                 'owner': '',
             }
 
             # TODO: move to the taskcluster package
-            def tasks(kind):
-                kind_path = mozpath.join(self.topsrcdir, 'taskcluster', 'ci', kind)
-                with open(mozpath.join(kind_path, 'kind.yml')) as f:
-                    config = yaml.load(f)
-                    tasks = Kind(kind, kind_path, config).load_tasks(params, {})
-                    return {
-                        task.task['metadata']['name']: task
-                        for task in tasks
-                    }
+            def tasks(kind_name):
+                root_path = mozpath.join(self.topsrcdir, 'taskcluster', 'ci')
+                graph_config = load_graph_config(root_path)
+                tasks = Kind.load(root_path, graph_config, kind_name).load_tasks(params, {})
+                return {
+                    task.task['metadata']['name']: task
+                    for task in tasks
+                }
 
             toolchains = tasks('toolchain')
 
             aliases = {}
             for t in toolchains.values():
                 alias = t.attributes.get('toolchain-alias')
                 if alias:
                     aliases['toolchain-{}'.format(alias)] = \
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -20,16 +20,22 @@ from .util.verify import (
     verify_docs,
     verifications,
 )
 from .config import validate_graph_config
 
 logger = logging.getLogger(__name__)
 
 
+class KindNotFound(Exception):
+    """
+    Raised when trying to load kind from a directory without a kind.yml.
+    """
+
+
 class Kind(object):
 
     def __init__(self, name, path, config, graph_config):
         self.name = name
         self.path = path
         self.config = config
         self.graph_config = graph_config
 
@@ -62,16 +68,41 @@ class Kind(object):
                       label=task_dict['label'],
                       attributes=task_dict['attributes'],
                       task=task_dict['task'],
                       optimization=task_dict.get('optimization'),
                       dependencies=task_dict.get('dependencies'))
                  for task_dict in transforms(trans_config, inputs)]
         return tasks
 
+    @classmethod
+    def load(cls, root_dir, graph_config, kind_name):
+        path = os.path.join(root_dir, kind_name)
+        kind_yml = os.path.join(path, 'kind.yml')
+        if not os.path.exists(kind_yml):
+            raise KindNotFound(kind_yml)
+
+        logger.debug("loading kind `{}` from `{}`".format(kind_name, path))
+        with open(kind_yml) as f:
+            config = yaml.load(f)
+
+        return cls(kind_name, path, config, graph_config)
+
+
+def load_graph_config(root_dir):
+    config_yml = os.path.join(root_dir, "config.yml")
+    if not os.path.exists(config_yml):
+        raise Exception("Couldn't find taskgraph configuration: {}".format(config_yml))
+
+    logger.debug("loading config from `{}`".format(config_yml))
+    with open(config_yml) as f:
+        config = yaml.load(f)
+
+    return validate_graph_config(config)
+
 
 class TaskGraphGenerator(object):
     """
     The central controller for taskgraph.  This handles all phases of graph
     generation.  The task is generated from all of the kinds defined in
     subdirectories of the generator's root directory.
 
     Access to the results of this generation, as well as intermediate values at
@@ -177,46 +208,25 @@ class TaskGraphGenerator(object):
         will have the same meaning as the optimized task graph, but be in a form
         more palatable to TaskCluster.
 
         @type: TaskGraph
         """
         return self._run_until('morphed_task_graph')
 
     def _load_kinds(self, graph_config):
-        for path in os.listdir(self.root_dir):
-            path = os.path.join(self.root_dir, path)
-            if not os.path.isdir(path):
-                continue
-            kind_name = os.path.basename(path)
-
-            kind_yml = os.path.join(path, 'kind.yml')
-            if not os.path.exists(kind_yml):
+        for kind_name in os.listdir(self.root_dir):
+            try:
+                yield Kind.load(self.root_dir, graph_config, kind_name)
+            except KindNotFound:
                 continue
 
-            logger.debug("loading kind `{}` from `{}`".format(kind_name, path))
-            with open(kind_yml) as f:
-                config = yaml.load(f)
-
-            yield Kind(kind_name, path, config, graph_config)
-
-    def _load_graph_config(self):
-        config_yml = os.path.join(self.root_dir, "config.yml")
-        if not os.path.exists(config_yml):
-            raise Exception("Couldn't find taskgraph configuration: {}".format(config_yml))
-
-        logger.debug("loading config from `{}`".format(config_yml))
-        with open(config_yml) as f:
-            config = yaml.load(f)
-
-        return validate_graph_config(config)
-
     def _run(self):
         logger.info("Loading graph configuration.")
-        graph_config = self._load_graph_config()
+        graph_config = load_graph_config(self.root_dir)
 
         logger.info("Loading kinds")
         # put the kinds into a graph and sort topologically so that kinds are loaded
         # in post-order
         kinds = {kind.name: kind for kind in self._load_kinds(graph_config)}
         self.verify_kinds(kinds)
 
         edges = set()
--- a/taskcluster/taskgraph/test/test_generator.py
+++ b/taskcluster/taskgraph/test/test_generator.py
@@ -7,16 +7,17 @@ from __future__ import absolute_import, 
 import pytest
 import unittest
 from mozunit import main
 
 from taskgraph.generator import TaskGraphGenerator, Kind
 from taskgraph.optimize import OptimizationStrategy
 from taskgraph.util.templates import merge
 from taskgraph import (
+    generator,
     graph,
     optimize as optimize_mod,
     target_tasks as target_tasks_mod,
 )
 
 
 def fake_loader(kind, path, config, parameters, loaded_tasks):
     for i in range(3):
@@ -52,18 +53,19 @@ class WithFakeKind(TaskGraphGenerator):
         for kind_name, cfg in self.parameters['_kinds']:
             config = {
                 'transforms': [],
             }
             if cfg:
                 config.update(cfg)
             yield FakeKind(kind_name, '/fake', config, graph_config)
 
-    def _load_graph_config(self):
-        return {}
+
+def fake_load_graph_config(root_dir):
+    return {}
 
 
 class FakeParameters(dict):
     strict = True
 
 
 class FakeOptimization(OptimizationStrategy):
     def __init__(self, mode, *args, **kwargs):
@@ -103,16 +105,18 @@ class TestGenerator(unittest.TestCase):
 
         parameters = FakeParameters({
             '_kinds': kinds,
             'target_tasks_method': 'test_method',
             'try_mode': None,
         })
         parameters.update(params)
 
+        self.patch.setattr(generator, 'load_graph_config', fake_load_graph_config)
+
         return WithFakeKind('/root', parameters)
 
     def test_kind_ordering(self):
         "When task kinds depend on each other, they are loaded in postorder"
         self.tgg = self.maketgg(kinds=[
             ('_fake3', {'kind-dependencies': ['_fake2', '_fake1']}),
             ('_fake2', {'kind-dependencies': ['_fake1']}),
             ('_fake1', {'kind-dependencies': []}),