Bug 1568277 - [taskgraph] Move 'taskgraph.transforms.job.import_all' to a utility function r=tomprince
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 15 Aug 2019 18:48:30 +0000
changeset 488334 cb35fd836621e5d1139149d1ab5485b01c8df781
parent 488333 c8f797a197313295516bfa1b05962ea54708e997
child 488335 d2b1d6c0a7322cc546c885dc164e812b693878aa
push id113906
push userncsoregi@mozilla.com
push dateFri, 16 Aug 2019 04:07:24 +0000
treeherdermozilla-inbound@d887276421d3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1568277 - [taskgraph] Move 'taskgraph.transforms.job.import_all' to a utility function r=tomprince Differential Revision: https://phabricator.services.mozilla.com/D41011
--- a/taskcluster/taskgraph/actions/registry.py
+++ b/taskcluster/taskgraph/actions/registry.py
@@ -13,16 +13,17 @@ from slugid import nice as slugid
 from types import FunctionType
 from collections import namedtuple
 from six import text_type
 from taskgraph import create
 from taskgraph.config import load_graph_config
 from taskgraph.util import taskcluster, yaml, hash
+from taskgraph.util.python_path import import_sibling_modules
 from taskgraph.parameters import Parameters
 from mozbuild.util import memoize
 actions = []
 callbacks = {}
 Action = namedtuple('Action', ['order', 'cb_name', 'generic', 'action_builder'])
@@ -317,20 +318,17 @@ def trigger_action_callback(task_group_i
         sanity_check_task_scope(callback, parameters, graph_config)
     cb(Parameters(**parameters), graph_config, input, task_group_id, task_id)
 def _load(graph_config):
     # Load all modules from this folder, relying on the side-effects of register_
     # functions to populate the action registry.
-    actions_dir = os.path.dirname(__file__)
-    for f in os.listdir(actions_dir):
-        if f.endswith('.py') and f not in ('__init__.py', 'registry.py', 'util.py'):
-            __import__('taskgraph.actions.' + f[:-3])
+    import_sibling_modules(exceptions=('util.py',))
     return callbacks, actions
 def _get_callbacks(graph_config):
     return _load(graph_config)[0]
 def _get_actions(graph_config):
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -9,25 +9,25 @@ the job at a higher level, using a "run"
 run-using handlers in `taskcluster/taskgraph/transforms/job`.
 from __future__ import absolute_import, print_function, unicode_literals
 import copy
 import logging
 import json
-import os
 import mozpack.path as mozpath
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import (
+from taskgraph.util.python_path import import_sibling_modules
 from taskgraph.util.taskcluster import get_artifact_prefix
 from taskgraph.util.workertypes import worker_type_implementation
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import (
@@ -252,17 +252,18 @@ def use_fetches(config, jobs):
         yield job
 def make_task_description(config, jobs):
     """Given a build description, create a task description"""
     # import plugin modules first, before iterating over jobs
-    import_all()
+    import_sibling_modules(exceptions=('common.py',))
     for job in jobs:
         if 'label' not in job:
             if 'name' not in job:
                 raise Exception("job has neither a name nor a label")
             job['label'] = '{}-{}'.format(config.kind, job['name'])
         if job.get('name'):
             del job['name']
@@ -338,16 +339,8 @@ def configure_taskdesc_for_run(config, j
         job['run'].setdefault(k, v)
     if schema:
                 schema, job['run'],
                 "In job.run using {!r}/{!r} for job {!r}:".format(
                     job['run']['using'], worker_implementation, job['label']))
     func(config, job, taskdesc)
-def import_all():
-    """Import all modules that are siblings of this one, triggering the decorator
-    above in the process."""
-    for f in os.listdir(os.path.dirname(__file__)):
-        if f.endswith('.py') and f not in ('commmon.py', '__init__.py'):
-            __import__('taskgraph.transforms.job.' + f[:-3])
--- a/taskcluster/taskgraph/util/python_path.py
+++ b/taskcluster/taskgraph/util/python_path.py
@@ -1,14 +1,17 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 from __future__ import absolute_import, print_function, unicode_literals
+import inspect
+import os
 def find_object(path):
     Find a Python object given a path of the form <modulepath>:<objectpath>.
     Conceptually equivalent to
         def find_object(modulepath, objectpath):
             import <modulepath> as mod
@@ -20,8 +23,33 @@ def find_object(path):
     modulepath, objectpath = path.split(':')
     obj = __import__(modulepath)
     for a in modulepath.split('.')[1:]:
         obj = getattr(obj, a)
     for a in objectpath.split('.'):
         obj = getattr(obj, a)
     return obj
+def import_sibling_modules(exceptions=None):
+    """
+    Import all Python modules that are siblings of the calling module.
+    Args:
+        exceptions (list): A list of file names to exclude (caller and
+            __init__.py are implicitly excluded).
+    """
+    frame = inspect.stack()[1]
+    mod = inspect.getmodule(frame[0])
+    name = os.path.basename(mod.__file__)
+    excs = set(['__init__.py', name])
+    if exceptions:
+        excs.update(exceptions)
+    modpath = mod.__name__
+    if not name.startswith('__init__.py'):
+        modpath = modpath.rsplit('.', 1)[0]
+    for f in os.listdir(os.path.dirname(mod.__file__)):
+        if f.endswith('.py') and f not in excs:
+            __import__(modpath + '.' + f[:-3])