Backed out 10 changesets (bug 1568277) for breaking ./mach bootstrap. a=backout
authorSebastian Hengst <archaeopteryx@coole-files.de>
Fri, 16 Aug 2019 11:19:54 +0200
changeset 488476 8da8443e0bcb7a6d9766d179332443660c926d8b
parent 488441 d887276421d30c773431c006b0607cd30f4d9259
child 488477 5d4cbfe103bbc517599231eb33d4f3ebbbcede40
push id92771
push userccoroiu@mozilla.com
push dateFri, 16 Aug 2019 09:55:21 +0000
treeherderautoland@43423b54465d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbackout
bugs1568277
milestone70.0a1
backs out056d9515483cd593dc1a931b4ade7ba7b143b540
632d943c947b359b68d7d29626ef3c51744572fd
d7e8f80e2c85288a5f59135f147475fe7ac73ce0
7b59ed5d703d3a9d6eae1860487da01a697106dd
f8b41cbaaf8e4f5205aa3d429bd56ccd36a4ace1
391a90f3f02babc4ef8a79b99b4a5a838a8b18ea
d2b1d6c0a7322cc546c885dc164e812b693878aa
cb35fd836621e5d1139149d1ab5485b01c8df781
c8f797a197313295516bfa1b05962ea54708e997
9b89f970d46b0c966e7f523171cee853cb47407c
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out 10 changesets (bug 1568277) for breaking ./mach bootstrap. a=backout Backed out changeset 056d9515483c (bug 1568277) Backed out changeset 632d943c947b (bug 1568277) Backed out changeset d7e8f80e2c85 (bug 1568277) Backed out changeset 7b59ed5d703d (bug 1568277) Backed out changeset f8b41cbaaf8e (bug 1568277) Backed out changeset 391a90f3f02b (bug 1568277) Backed out changeset d2b1d6c0a732 (bug 1568277) Backed out changeset cb35fd836621 (bug 1568277) Backed out changeset c8f797a19731 (bug 1568277) Backed out changeset 9b89f970d46b (bug 1568277)
taskcluster/ci/config.yml
taskcluster/ci/source-test/kind.yml
taskcluster/ci/source-test/shadow-scheduler.yml
taskcluster/mach_commands.py
taskcluster/taskgraph/actions/registry.py
taskcluster/taskgraph/generator.py
taskcluster/taskgraph/optimize.py
taskcluster/taskgraph/optimize/__init__.py
taskcluster/taskgraph/optimize/seta.py
taskcluster/taskgraph/optimize/strategies.py
taskcluster/taskgraph/test/test_generator.py
taskcluster/taskgraph/test/test_optimize.py
taskcluster/taskgraph/transforms/job/__init__.py
taskcluster/taskgraph/transforms/source_test.py
taskcluster/taskgraph/transforms/tests.py
taskcluster/taskgraph/util/python_path.py
taskcluster/taskgraph/util/schema.py
taskcluster/taskgraph/util/seta.py
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -110,17 +110,16 @@ treeherder:
         'langpack': 'Langpack sigatures and uploads'
         'TPS': 'Sync tests'
         'UV': 'Update verify'
         'pipfu': 'pipfile update'
         'WR': 'WebRender standalone'
         'Gd': 'Geckodriver'
         'clang': 'Clang Tidy & Format'
         'coverity': 'Coverity Static Analysis'
-        'SS': 'Shadow scheduler'
 
 index:
     products:
         - 'firefox'
         - 'fennec'
         - 'mobile'
         - 'static-analysis'
         - 'devedition'
--- a/taskcluster/ci/source-test/kind.yml
+++ b/taskcluster/ci/source-test/kind.yml
@@ -21,17 +21,16 @@ jobs-from:
     - doc.yml
     - file-metadata.yml
     - infer.yml
     - jsshell.yml
     - mozlint.yml
     - mozlint-android.yml
     - node.yml
     - python.yml
-    - shadow-scheduler.yml
     - webidl.yml
     - wpt-metadata.yml
     - wpt-manifest.yml
 
 job-defaults:
     attributes:
         retrigger: true
 
deleted file mode 100644
--- a/taskcluster/ci/source-test/shadow-scheduler.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
----
-job-defaults:
-    platform: gecko-decision/opt
-    worker-type: t-linux-xlarge
-    worker:
-        docker-image: {in-tree: "lint"}
-        max-run-time: 3600
-        artifacts:
-            - type: file
-              name: public/shadow-scheduler/optimized_tasks.list
-              path: /builds/worker/optimized_tasks.list
-    treeherder:
-        kind: other
-        tier: 3
-    require-decision-task-id: true
-    run-on-projects: ['autoland']
-    run:
-        using: mach
-        mach: taskgraph optimized -p task-id=$DECISION_TASK_ID --output-file /builds/worker/optimized_tasks.list
-        sparse-profile: taskgraph
-
-seta_10_120:
-    description: Runs the seta_10_120 experimental optimization strategy
-    treeherder:
-        symbol: SS(seta_10_120)
-    worker:
-        env:
-            TASKGRAPH_OPTIMIZE_STRATEGIES: taskgraph.optimize:seta_10_120
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -54,18 +54,16 @@ class ShowTaskGraphSubCommand(SubCommand
             CommandArgument('--tasks-regex', '--tasks', default=None,
                             help="only return tasks with labels matching this regular "
                             "expression."),
             CommandArgument('--target-kind', default=None,
                             help="only return tasks that are of the given kind, "
                                  "or their dependencies."),
             CommandArgument('-F', '--fast', dest='fast', default=False, action='store_true',
                             help="enable fast task generation for local debugging."),
-            CommandArgument('-o', '--output-file', default=None,
-                            help="file path to store generated output."),
 
         ]
         for arg in args:
             after = arg(after)
         return after
 
 
 @CommandProvider
@@ -375,33 +373,28 @@ class MachCommands(MachCommandBase):
                 parameters=parameters,
                 target_kind=options.get('target_kind'),
             )
 
             tg = getattr(tgg, graph_attr)
 
             show_method = getattr(self, 'show_taskgraph_' + (options['format'] or 'labels'))
             tg = self.get_filtered_taskgraph(tg, options["tasks_regex"])
-
-            fh = options['output_file']
-            if fh:
-                fh = open(fh, 'w')
-            show_method(tg, file=fh)
+            show_method(tg)
         except Exception:
             traceback.print_exc()
             sys.exit(1)
 
-    def show_taskgraph_labels(self, taskgraph, file=None):
+    def show_taskgraph_labels(self, taskgraph):
         for index in taskgraph.graph.visit_postorder():
-            print(taskgraph.tasks[index].label, file=file)
+            print(taskgraph.tasks[index].label)
 
-    def show_taskgraph_json(self, taskgraph, file=None):
+    def show_taskgraph_json(self, taskgraph):
         print(json.dumps(taskgraph.to_json(),
-              sort_keys=True, indent=2, separators=(',', ': ')),
-              file=file)
+              sort_keys=True, indent=2, separators=(',', ': ')))
 
     def get_filtered_taskgraph(self, taskgraph, tasksregex):
         from taskgraph.graph import Graph
         from taskgraph.taskgraph import TaskGraph
         """
         This class method filters all the tasks on basis of a regular expression
         and returns a new TaskGraph object
         """
--- a/taskcluster/taskgraph/actions/registry.py
+++ b/taskcluster/taskgraph/actions/registry.py
@@ -13,17 +13,16 @@ from slugid import nice as slugid
 from types import FunctionType
 from collections import namedtuple
 
 from six import text_type
 
 from taskgraph import create
 from taskgraph.config import load_graph_config
 from taskgraph.util import taskcluster, yaml, hash
-from taskgraph.util.python_path import import_sibling_modules
 from taskgraph.parameters import Parameters
 from mozbuild.util import memoize
 
 
 actions = []
 callbacks = {}
 
 Action = namedtuple('Action', ['order', 'cb_name', 'generic', 'action_builder'])
@@ -318,17 +317,20 @@ def trigger_action_callback(task_group_i
         sanity_check_task_scope(callback, parameters, graph_config)
 
     cb(Parameters(**parameters), graph_config, input, task_group_id, task_id)
 
 
 def _load(graph_config):
     # Load all modules from this folder, relying on the side-effects of register_
     # functions to populate the action registry.
-    import_sibling_modules(exceptions=('util.py',))
+    actions_dir = os.path.dirname(__file__)
+    for f in os.listdir(actions_dir):
+        if f.endswith('.py') and f not in ('__init__.py', 'registry.py', 'util.py'):
+            __import__('taskgraph.actions.' + f[:-3])
     return callbacks, actions
 
 
 def _get_callbacks(graph_config):
     return _load(graph_config)[0]
 
 
 def _get_actions(graph_config):
--- a/taskcluster/taskgraph/generator.py
+++ b/taskcluster/taskgraph/generator.py
@@ -329,17 +329,17 @@ class TaskGraphGenerator(object):
         if strategies:
             strategies = find_object(strategies)
 
         optimized_task_graph, label_to_taskid = optimize_task_graph(
             target_task_graph,
             parameters,
             do_not_optimize,
             existing_tasks=existing_tasks,
-            strategy_override=strategies,
+            strategies=strategies,
         )
 
         yield verifications('optimized_task_graph', optimized_task_graph, graph_config)
 
         morphed_task_graph, label_to_taskid = morph(
             optimized_task_graph, label_to_taskid, parameters)
 
         yield 'label_to_taskid', label_to_taskid
rename from taskcluster/taskgraph/optimize/__init__.py
rename to taskcluster/taskgraph/optimize.py
--- a/taskcluster/taskgraph/optimize/__init__.py
+++ b/taskcluster/taskgraph/optimize.py
@@ -9,51 +9,48 @@ a push, then the resulting graph contain
 task.
 
 See ``taskcluster/docs/optimization.rst`` for more information.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import logging
+import os
 from collections import defaultdict
 
+from .graph import Graph
+from . import files_changed
+from .taskgraph import TaskGraph
+from .util.seta import is_low_value_task
+from .util.perfile import perfile_number_of_chunks
+from .util.taskcluster import find_task_id
+from .util.parameterization import resolve_task_references
+from mozbuild.util import memoize
 from slugid import nice as slugid
-
-from taskgraph.graph import Graph
-from taskgraph.taskgraph import TaskGraph
-from taskgraph.util.parameterization import resolve_task_references
-from taskgraph.util.python_path import import_sibling_modules
+from mozbuild.base import MozbuildObject
 
 logger = logging.getLogger(__name__)
-registry = {}
 
-
-def register_strategy(name, args=()):
-    def wrap(cls):
-        if name not in registry:
-            registry[name] = cls(*args)
-        return cls
-    return wrap
+TOPSRCDIR = os.path.abspath(os.path.join(__file__, '../../../'))
 
 
 def optimize_task_graph(target_task_graph, params, do_not_optimize,
-                        existing_tasks=None, strategy_override=None):
+                        existing_tasks=None, strategies=None):
     """
     Perform task optimization, returning a taskgraph and a map from label to
     assigned taskId, including replacement tasks.
     """
     label_to_taskid = {}
     if not existing_tasks:
         existing_tasks = {}
 
     # instantiate the strategies for this optimization process
-    strategies = registry.copy()
-    if strategy_override:
-        strategies.update(strategy_override)
+    if not strategies:
+        strategies = _make_default_strategies()
 
     optimizations = _get_optimizations(target_task_graph, strategies)
 
     removed_tasks = remove_tasks(
         target_task_graph=target_task_graph,
         optimizations=optimizations,
         params=params,
         do_not_optimize=do_not_optimize)
@@ -67,25 +64,33 @@ def optimize_task_graph(target_task_grap
         existing_tasks=existing_tasks,
         removed_tasks=removed_tasks)
 
     return get_subgraph(
             target_task_graph, removed_tasks, replaced_tasks,
             label_to_taskid), label_to_taskid
 
 
+def _make_default_strategies():
+    return {
+        'never': OptimizationStrategy(),  # "never" is the default behavior
+        'index-search': IndexSearch(),
+        'seta': SETA(),
+        'skip-unless-changed': SkipUnlessChanged(),
+        'skip-unless-schedules': SkipUnlessSchedules(),
+        'skip-unless-schedules-or-seta': Either(SkipUnlessSchedules(), SETA()),
+    }
+
+
 def _get_optimizations(target_task_graph, strategies):
     def optimizations(label):
         task = target_task_graph.tasks[label]
         if task.optimization:
             opt_by, arg = task.optimization.items()[0]
-            strategy = strategies[opt_by]
-            if hasattr(strategy, 'description'):
-                opt_by += " ({})".format(strategy.description)
-            return (opt_by, strategy, arg)
+            return (opt_by, strategies[opt_by], arg)
         else:
             return ('never', strategies['never'], None)
     return optimizations
 
 
 def _log_optimization(verb, opt_counts):
     if opt_counts:
         logger.info(
@@ -236,17 +241,16 @@ def get_subgraph(target_task_graph, remo
         if left in tasks_by_taskid and right in tasks_by_taskid
     )
 
     return TaskGraph(
         tasks_by_taskid,
         Graph(set(tasks_by_taskid), edges_by_taskid))
 
 
-@register_strategy('never')
 class OptimizationStrategy(object):
     def should_remove_task(self, task, params, arg):
         """Determine whether to optimize this task by removing it.  Returns
         True to remove."""
         return False
 
     def should_replace_task(self, task, params, arg):
         """Determine whether to optimize this task by replacing it.  Returns a
@@ -256,23 +260,17 @@ class OptimizationStrategy(object):
 
 
 class Either(OptimizationStrategy):
     """Given one or more optimization strategies, remove a task if any of them
     says to, and replace with a task if any finds a replacement (preferring the
     earliest).  By default, each substrategy gets the same arg, but split_args
     can return a list of args for each strategy, if desired."""
     def __init__(self, *substrategies, **kwargs):
-        missing = set(substrategies) - set(registry.keys())
-        if missing:
-            raise TypeError("substrategies aren't registered: {}".format(
-                ",  ".join(sorted(missing))))
-
-        self.description = "-or-".join(substrategies)
-        self.substrategies = [registry[sub] for sub in substrategies]
+        self.substrategies = substrategies
         self.split_args = kwargs.pop('split_args', None)
         if not self.split_args:
             self.split_args = lambda arg: [arg] * len(substrategies)
         if kwargs:
             raise TypeError("unexpected keyword args")
 
     def _for_substrategies(self, arg, fn):
         for sub, arg in zip(self.substrategies, self.split_args(arg)):
@@ -287,34 +285,109 @@ class Either(OptimizationStrategy):
             lambda sub, arg: sub.should_remove_task(task, params, arg))
 
     def should_replace_task(self, task, params, arg):
         return self._for_substrategies(
             arg,
             lambda sub, arg: sub.should_replace_task(task, params, arg))
 
 
-class Alias(Either):
-    """Provides an alias to an existing strategy.
+class IndexSearch(OptimizationStrategy):
+
+    # A task with no dependencies remaining after optimization will be replaced
+    # if artifacts exist for the corresponding index_paths.
+    # Otherwise, we're in one of the following cases:
+    # - the task has un-optimized dependencies
+    # - the artifacts have expired
+    # - some changes altered the index_paths and new artifacts need to be
+    # created.
+    # In every of those cases, we need to run the task to create or refresh
+    # artifacts.
 
-    This can be useful to swap strategies in and out without needing to modify
-    the task transforms.
-    """
-    def __init__(self, strategy):
-        super(Alias, self).__init__(strategy)
+    def should_replace_task(self, task, params, index_paths):
+        "Look for a task with one of the given index paths"
+        for index_path in index_paths:
+            try:
+                task_id = find_task_id(
+                    index_path,
+                    use_proxy=bool(os.environ.get('TASK_ID')))
+                return task_id
+            except KeyError:
+                # 404 will end up here and go on to the next index path
+                pass
+
+        return False
+
+
+class SETA(OptimizationStrategy):
+    def should_remove_task(self, task, params, _):
+        label = task.label
+
+        # we would like to return 'False, None' while it's high_value_task
+        # and we wouldn't optimize it. Otherwise, it will return 'True, None'
+        if is_low_value_task(label,
+                             params.get('project'),
+                             params.get('pushlog_id'),
+                             params.get('pushdate')):
+            # Always optimize away low-value tasks
+            return True
+        else:
+            return False
 
 
-# Trigger registration in sibling modules.
-import_sibling_modules()
+class SkipUnlessChanged(OptimizationStrategy):
+    def should_remove_task(self, task, params, file_patterns):
+        # pushlog_id == -1 - this is the case when run from a cron.yml job
+        if params.get('pushlog_id') == -1:
+            return False
+
+        changed = files_changed.check(params, file_patterns)
+        if not changed:
+            logger.debug('no files found matching a pattern in `skip-unless-changed` for ' +
+                         task.label)
+            return True
+        return False
 
 
-# Register composite strategies.
-register_strategy('test', args=('skip-unless-schedules', 'seta'))(Either)
-register_strategy('test-inclusive', args=('skip-unless-schedules',))(Alias)
-register_strategy('test-try', args=('skip-unless-schedules',))(Alias)
+class SkipUnlessSchedules(OptimizationStrategy):
+
+    @memoize
+    def scheduled_by_push(self, repository, revision):
+        changed_files = files_changed.get_changed_files(repository, revision)
+
+        mbo = MozbuildObject.from_environment()
+        # the decision task has a sparse checkout, so, mozbuild_reader will use
+        # a MercurialRevisionFinder with revision '.', which should be the same
+        # as `revision`; in other circumstances, it will use a default reader
+        rdr = mbo.mozbuild_reader(config_mode='empty')
+
+        components = set()
+        for p, m in rdr.files_info(changed_files).items():
+            components |= set(m['SCHEDULES'].components)
+
+        return components
+
+    def should_remove_task(self, task, params, conditions):
+        if params.get('pushlog_id') == -1:
+            return False
+
+        scheduled = self.scheduled_by_push(params['head_repository'], params['head_rev'])
+        conditions = set(conditions)
+        # if *any* of the condition components are scheduled, do not optimize
+        if conditions & scheduled:
+            return False
+
+        return True
 
 
-# Experimental strategy that replaces the default SETA with a version that runs
-# all tasks every 10th push or 2 hours.
-seta_10_120 = {
-    'seta': Alias('seta_10_120'),
-    'test': Either('skip-unless-schedules', 'seta_10_120'),
-}
+class TestVerify(OptimizationStrategy):
+    def should_remove_task(self, task, params, _):
+        # we would like to return 'False, None' while it's high_value_task
+        # and we wouldn't optimize it. Otherwise, it will return 'True, None'
+        env = params.get('try_task_config', {}) or {}
+        env = env.get('templates', {}).get('env', {})
+        if perfile_number_of_chunks(params.is_try(),
+                                    env.get('MOZHARNESS_TEST_PATHS', ''),
+                                    params.get('head_repository', ''),
+                                    params.get('head_rev', ''),
+                                    task):
+            return False
+        return True
deleted file mode 100644
--- a/taskcluster/taskgraph/optimize/strategies.py
+++ /dev/null
@@ -1,92 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-from __future__ import absolute_import, print_function, unicode_literals
-
-import logging
-import os
-
-from mozbuild.base import MozbuildObject
-from mozbuild.util import memoize
-
-from taskgraph import files_changed
-from taskgraph.optimize import register_strategy, OptimizationStrategy
-from taskgraph.util.taskcluster import find_task_id
-
-logger = logging.getLogger(__name__)
-
-
-@register_strategy("index-search")
-class IndexSearch(OptimizationStrategy):
-
-    # A task with no dependencies remaining after optimization will be replaced
-    # if artifacts exist for the corresponding index_paths.
-    # Otherwise, we're in one of the following cases:
-    # - the task has un-optimized dependencies
-    # - the artifacts have expired
-    # - some changes altered the index_paths and new artifacts need to be
-    # created.
-    # In every of those cases, we need to run the task to create or refresh
-    # artifacts.
-
-    def should_replace_task(self, task, params, index_paths):
-        "Look for a task with one of the given index paths"
-        for index_path in index_paths:
-            try:
-                task_id = find_task_id(
-                    index_path,
-                    use_proxy=bool(os.environ.get('TASK_ID')))
-                return task_id
-            except KeyError:
-                # 404 will end up here and go on to the next index path
-                pass
-
-        return False
-
-
-@register_strategy("skip-unless-changed")
-class SkipUnlessChanged(OptimizationStrategy):
-    def should_remove_task(self, task, params, file_patterns):
-        # pushlog_id == -1 - this is the case when run from a cron.yml job
-        if params.get('pushlog_id') == -1:
-            return False
-
-        changed = files_changed.check(params, file_patterns)
-        if not changed:
-            logger.debug('no files found matching a pattern in `skip-unless-changed` for ' +
-                         task.label)
-            return True
-        return False
-
-
-@register_strategy("skip-unless-schedules")
-class SkipUnlessSchedules(OptimizationStrategy):
-
-    @memoize
-    def scheduled_by_push(self, repository, revision):
-        changed_files = files_changed.get_changed_files(repository, revision)
-
-        mbo = MozbuildObject.from_environment()
-        # the decision task has a sparse checkout, so, mozbuild_reader will use
-        # a MercurialRevisionFinder with revision '.', which should be the same
-        # as `revision`; in other circumstances, it will use a default reader
-        rdr = mbo.mozbuild_reader(config_mode='empty')
-
-        components = set()
-        for p, m in rdr.files_info(changed_files).items():
-            components |= set(m['SCHEDULES'].components)
-
-        return components
-
-    def should_remove_task(self, task, params, conditions):
-        if params.get('pushlog_id') == -1:
-            return False
-
-        scheduled = self.scheduled_by_push(params['head_repository'], params['head_rev'])
-        conditions = set(conditions)
-        # if *any* of the condition components are scheduled, do not optimize
-        if conditions & scheduled:
-            return False
-
-        return True
--- a/taskcluster/taskgraph/test/test_generator.py
+++ b/taskcluster/taskgraph/test/test_generator.py
@@ -91,21 +91,22 @@ class TestGenerator(unittest.TestCase):
     def maketgg(self, target_tasks=None, kinds=[('_fake', [])], params=None):
         params = params or {}
         FakeKind.loaded_kinds = []
         self.target_tasks = target_tasks or []
 
         def target_tasks_method(full_task_graph, parameters, graph_config):
             return self.target_tasks
 
-        fake_registry = {mode: FakeOptimization(mode)
-                         for mode in ('always', 'never', 'even', 'odd')}
+        def make_fake_strategies():
+            return {mode: FakeOptimization(mode)
+                    for mode in ('always', 'never', 'even', 'odd')}
 
         target_tasks_mod._target_task_methods['test_method'] = target_tasks_method
-        self.patch.setattr(optimize_mod, 'registry', fake_registry)
+        self.patch.setattr(optimize_mod, '_make_default_strategies', make_fake_strategies)
 
         parameters = FakeParameters({
             '_kinds': kinds,
             'target_tasks_method': 'test_method',
             'try_mode': None,
         })
         parameters.update(params)
 
--- a/taskcluster/taskgraph/test/test_optimize.py
+++ b/taskcluster/taskgraph/test/test_optimize.py
@@ -1,40 +1,40 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import unittest
 
-from taskgraph import graph, optimize
-from taskgraph.optimize import OptimizationStrategy
+from taskgraph import optimize
 from taskgraph.taskgraph import TaskGraph
+from taskgraph import graph
 from taskgraph.task import Task
 from mozunit import main
 from slugid import nice as slugid
 
 
-class Remove(OptimizationStrategy):
+class Remove(optimize.OptimizationStrategy):
 
     def should_remove_task(self, task, params, arg):
         return True
 
 
-class Replace(OptimizationStrategy):
+class Replace(optimize.OptimizationStrategy):
 
     def should_replace_task(self, task, params, taskid):
         return taskid
 
 
 class TestOptimize(unittest.TestCase):
 
     strategies = {
-        'never': OptimizationStrategy(),
+        'never': optimize.OptimizationStrategy(),
         'remove': Remove(),
         'replace': Replace(),
     }
 
     def make_task(self, label, optimization=None, task_def=None, optimized=None,
                   task_id=None, dependencies=None):
         task_def = task_def or {'sample': 'task-def'}
         task = Task(kind='test', label=label, attributes={}, task=task_def)
@@ -65,17 +65,16 @@ class TestOptimize(unittest.TestCase):
             self.make_task('t1', opts.get('t1')),
             self.make_task('t2', opts.get('t2')),
             self.make_task('t3', opts.get('t3')),
             ('t3', 't2', 'dep'),
             ('t3', 't1', 'dep2'),
             ('t2', 't1', 'dep'))
 
     def assert_remove_tasks(self, graph, exp_removed, do_not_optimize=set()):
-        optimize.registry = self.strategies
         got_removed = optimize.remove_tasks(
             target_task_graph=graph,
             optimizations=optimize._get_optimizations(graph, self.strategies),
             params={},
             do_not_optimize=do_not_optimize)
         self.assertEqual(got_removed, exp_removed)
 
     def test_remove_tasks_never(self):
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -9,25 +9,25 @@ the job at a higher level, using a "run"
 run-using handlers in `taskcluster/taskgraph/transforms/job`.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import copy
 import logging
 import json
+import os
 
 import mozpack.path as mozpath
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import (
     validate_schema,
     Schema,
 )
-from taskgraph.util.python_path import import_sibling_modules
 from taskgraph.util.taskcluster import get_artifact_prefix
 from taskgraph.util.workertypes import worker_type_implementation
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import (
     Extra,
     Optional,
     Required,
     Exclusive,
@@ -252,18 +252,17 @@ def use_fetches(config, jobs):
 
         yield job
 
 
 @transforms.add
 def make_task_description(config, jobs):
     """Given a build description, create a task description"""
     # import plugin modules first, before iterating over jobs
-    import_sibling_modules(exceptions=('common.py',))
-
+    import_all()
     for job in jobs:
         if 'label' not in job:
             if 'name' not in job:
                 raise Exception("job has neither a name nor a label")
             job['label'] = '{}-{}'.format(config.kind, job['name'])
         if job.get('name'):
             del job['name']
 
@@ -339,8 +338,16 @@ def configure_taskdesc_for_run(config, j
         job['run'].setdefault(k, v)
 
     if schema:
         validate_schema(
                 schema, job['run'],
                 "In job.run using {!r}/{!r} for job {!r}:".format(
                     job['run']['using'], worker_implementation, job['label']))
     func(config, job, taskdesc)
+
+
+def import_all():
+    """Import all modules that are siblings of this one, triggering the decorator
+    above in the process."""
+    for f in os.listdir(os.path.dirname(__file__)):
+        if f.endswith('.py') and f not in ('commmon.py', '__init__.py'):
+            __import__('taskgraph.transforms.job.' + f[:-3])
--- a/taskcluster/taskgraph/transforms/source_test.py
+++ b/taskcluster/taskgraph/transforms/source_test.py
@@ -49,18 +49,16 @@ source_test_description_schema = Schema(
         job_description_schema['worker-type'],
         {'by-platform': {basestring: job_description_schema['worker-type']}},
     ),
     Required('worker'): Any(
         job_description_schema['worker'],
         {'by-platform': {basestring: job_description_schema['worker']}},
     ),
     Optional('python-version'): [int],
-    # If true, the DECISION_TASK_ID env will be populated.
-    Optional('require-decision-task-id'): bool,
 })
 
 transforms = TransformSequence()
 
 
 @transforms.add
 def set_defaults(config, jobs):
     for job in jobs:
@@ -210,24 +208,8 @@ def handle_shell(config, jobs):
             yield job
             continue
 
         for field in fields:
             resolve_keyed_by(job, field, item_name=job['name'])
 
         del job['shell']
         yield job
-
-
-@transforms.add
-def add_decision_task_id_to_env(config, jobs):
-    """
-    Creates the `DECISION_TASK_ID` environment variable in tasks that set the
-    `require-decision-task-id` config.
-    """
-    for job in jobs:
-        if not job.pop('require-decision-task-id', False):
-            yield job
-            continue
-
-        env = job['worker'].setdefault('env', {})
-        env['DECISION_TASK_ID'] = os.environ.get('TASK_ID', '')
-        yield job
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -1411,26 +1411,25 @@ def make_job_description(config, tests):
             # if this is an "inclusive" test, then all files which might
             # cause it to run are annotated with SCHEDULES in moz.build,
             # so do not include the platform or any other components here
             schedules = [category]
         else:
             schedules = [category, platform_family(test['build-platform'])]
 
         if test.get('when'):
-            # This may still be used by comm-central.
             jobdesc['when'] = test['when']
         elif 'optimization' in test:
             jobdesc['optimization'] = test['optimization']
-        elif config.params.is_try():
-            jobdesc['optimization'] = {'test-try': schedules}
-        elif category in INCLUSIVE_COMPONENTS:
-            jobdesc['optimization'] = {'test-inclusive': schedules}
+        elif not config.params.is_try() and category not in INCLUSIVE_COMPONENTS:
+            # for non-try branches and non-inclusive suites, include SETA
+            jobdesc['optimization'] = {'skip-unless-schedules-or-seta': schedules}
         else:
-            jobdesc['optimization'] = {'test': schedules}
+            # otherwise just use skip-unless-schedules
+            jobdesc['optimization'] = {'skip-unless-schedules': schedules}
 
         run = jobdesc['run'] = {}
         run['using'] = 'mozharness-test'
         run['test'] = test
 
         if 'workdir' in test:
             run['workdir'] = test.pop('workdir')
 
--- a/taskcluster/taskgraph/util/python_path.py
+++ b/taskcluster/taskgraph/util/python_path.py
@@ -1,17 +1,14 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
-import inspect
-import os
-
 
 def find_object(path):
     """
     Find a Python object given a path of the form <modulepath>:<objectpath>.
     Conceptually equivalent to
 
         def find_object(modulepath, objectpath):
             import <modulepath> as mod
@@ -23,33 +20,8 @@ def find_object(path):
 
     modulepath, objectpath = path.split(':')
     obj = __import__(modulepath)
     for a in modulepath.split('.')[1:]:
         obj = getattr(obj, a)
     for a in objectpath.split('.'):
         obj = getattr(obj, a)
     return obj
-
-
-def import_sibling_modules(exceptions=None):
-    """
-    Import all Python modules that are siblings of the calling module.
-
-    Args:
-        exceptions (list): A list of file names to exclude (caller and
-            __init__.py are implicitly excluded).
-    """
-    frame = inspect.stack()[1]
-    mod = inspect.getmodule(frame[0])
-
-    name = os.path.basename(mod.__file__)
-    excs = set(['__init__.py', name])
-    if exceptions:
-        excs.update(exceptions)
-
-    modpath = mod.__name__
-    if not name.startswith('__init__.py'):
-        modpath = modpath.rsplit('.', 1)[0]
-
-    for f in os.listdir(os.path.dirname(mod.__file__)):
-        if f.endswith('.py') and f not in excs:
-            __import__(modpath + '.' + f[:-3])
--- a/taskcluster/taskgraph/util/schema.py
+++ b/taskcluster/taskgraph/util/schema.py
@@ -198,20 +198,18 @@ OptimizationSchema = voluptuous.Any(
     # the search occurs in order, with the first match winning
     {'index-search': [basestring]},
     # consult SETA and skip this task if it is low-value
     {'seta': None},
     # skip this task if none of the given file patterns match
     {'skip-unless-changed': [basestring]},
     # skip this task if unless the change files' SCHEDULES contains any of these components
     {'skip-unless-schedules': list(schedules.ALL_COMPONENTS)},
-    # optimize strategy aliases for the test kind
-    {'test': list(schedules.ALL_COMPONENTS)},
-    {'test-inclusive': list(schedules.ALL_COMPONENTS)},
-    {'test-try': list(schedules.ALL_COMPONENTS)},
+    # skip if SETA or skip-unless-schedules says to
+    {'skip-unless-schedules-or-seta': list(schedules.ALL_COMPONENTS)},
 )
 
 # shortcut for a string where task references are allowed
 taskref_or_string = voluptuous.Any(
     basestring,
     {voluptuous.Required('task-reference'): basestring},
     {voluptuous.Required('artifact-reference'): basestring},
 )
rename from taskcluster/taskgraph/optimize/seta.py
rename to taskcluster/taskgraph/util/seta.py
--- a/taskcluster/taskgraph/optimize/seta.py
+++ b/taskcluster/taskgraph/util/seta.py
@@ -3,27 +3,26 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import logging
 import requests
 from collections import defaultdict
-
-import attr
 from redo import retry
 from requests import exceptions
-
-from taskgraph.optimize import OptimizationStrategy, register_strategy
+import attr
 
 logger = logging.getLogger(__name__)
 
 # It's a list of project name which SETA is useful on
 SETA_PROJECTS = ['mozilla-inbound', 'autoland']
+PROJECT_SCHEDULE_ALL_EVERY_PUSHES = {'mozilla-inbound': 5, 'autoland': 5}
+PROJECT_SCHEDULE_ALL_EVERY_MINUTES = {'mozilla-inbound': 60, 'autoland': 60}
 SETA_HIGH_PRIORITY = 1
 SETA_LOW_PRIORITY = 5
 
 SETA_ENDPOINT = "https://treeherder.mozilla.org/api/project/%s/seta/" \
                 "job-priorities/?build_system_type=%s&priority=%s"
 PUSH_ENDPOINT = "https://hg.mozilla.org/integration/%s/json-pushes/?startID=%d&endID=%d"
 
 
@@ -162,20 +161,20 @@ class SETA(object):
             logger.warning(error)
 
         # When we get invalid JSON (i.e. 500 error), it results in a ValueError (bug 1313426)
         except ValueError as error:
             logger.warning("Invalid JSON, possible server error: {}".format(error))
 
         return low_value_tasks
 
-    def minutes_between_pushes(self, project, cur_push_id, cur_push_date, time_interval):
+    def minutes_between_pushes(self, project, cur_push_id, cur_push_date):
         # figure out the minutes that have elapsed between the current push and previous one
         # defaulting to max min so if we can't get value, defaults to run the task
-        min_between_pushes = time_interval
+        min_between_pushes = PROJECT_SCHEDULE_ALL_EVERY_MINUTES.get(project, 60)
         prev_push_id = cur_push_id - 1
 
         # cache the pushdate for the current push so we can use it next time
         self.push_dates[project].update({cur_push_id: cur_push_date})
 
         # check if we already have the previous push id's datetime cached
         prev_push_date = self.push_dates[project].get(prev_push_id, 0)
 
@@ -228,61 +227,35 @@ class SETA(object):
 
         # We just print the error out as a debug message if we failed to catch the exception above
         except exceptions.RequestException as error:
             logger.warning(error)
             self.failed_json_push_calls.append(prev_push_id)
 
         return min_between_pushes
 
-    def is_low_value_task(self, label, project, pushlog_id, push_date,
-                          push_interval, time_interval):
+    def is_low_value_task(self, label, project, pushlog_id, push_date):
         # marking a task as low_value means it will be optimized out by tc
         if project not in SETA_PROJECTS:
             return False
 
+        schedule_all_every = PROJECT_SCHEDULE_ALL_EVERY_PUSHES.get(project, 5)
         # on every Nth push, want to run all tasks
-        if int(pushlog_id) % push_interval == 0:
+        if int(pushlog_id) % schedule_all_every == 0:
             return False
 
         # Nth push, so time to call seta based on number of pushes; however
         # we also want to ensure we run all tasks at least once per N minutes
         if self.minutes_between_pushes(
                 project,
                 int(pushlog_id),
-                int(push_date),
-                time_interval) >= time_interval:
+                int(push_date)) >= PROJECT_SCHEDULE_ALL_EVERY_MINUTES.get(project, 60):
             return False
 
         # cache the low value tasks per project to avoid repeated SETA server queries
         if project not in self.low_value_tasks:
             self.low_value_tasks[project] = self.query_low_value_tasks(project)
         return label in self.low_value_tasks[project]
 
 
 # create a single instance of this class, and expose its `is_low_value_task`
 # bound method as a module-level function
 is_low_value_task = SETA().is_low_value_task
-
-
-@register_strategy('seta', args=(5, 60))
-@register_strategy('seta_10_120', args=(10, 120))
-class SkipLowValue(OptimizationStrategy):
-
-    def __init__(self, push_interval, time_interval):
-        self.push_interval = push_interval
-        self.time_interval = time_interval
-
-    def should_remove_task(self, task, params, _):
-        label = task.label
-
-        # we would like to return 'False, None' while it's high_value_task
-        # and we wouldn't optimize it. Otherwise, it will return 'True, None'
-        if is_low_value_task(label,
-                             params.get('project'),
-                             params.get('pushlog_id'),
-                             params.get('pushdate'),
-                             self.push_interval,
-                             self.time_interval):
-            # Always optimize away low-value tasks
-            return True
-        else:
-            return False