Bug 1568277 - [taskgraph] Split optimize strategies out into a separate file r=tomprince
☠☠ backed out by 8da8443e0bcb ☠ ☠
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 15 Aug 2019 18:48:38 +0000
changeset 488334 d2b1d6c0a7322cc546c885dc164e812b693878aa
parent 488333 cb35fd836621e5d1139149d1ab5485b01c8df781
child 488335 391a90f3f02babc4ef8a79b99b4a5a838a8b18ea
push id36440
push userncsoregi@mozilla.com
push dateFri, 16 Aug 2019 03:57:48 +0000
treeherdermozilla-central@a58b7dc85887 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstomprince
bugs1568277
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1568277 - [taskgraph] Split optimize strategies out into a separate file r=tomprince Differential Revision: https://phabricator.services.mozilla.com/D40203
taskcluster/taskgraph/optimize.py
taskcluster/taskgraph/optimize/__init__.py
taskcluster/taskgraph/optimize/strategies.py
taskcluster/taskgraph/test/test_optimize.py
rename from taskcluster/taskgraph/optimize.py
rename to taskcluster/taskgraph/optimize/__init__.py
--- a/taskcluster/taskgraph/optimize.py
+++ b/taskcluster/taskgraph/optimize/__init__.py
@@ -9,29 +9,24 @@ a push, then the resulting graph contain
 task.
 
 See ``taskcluster/docs/optimization.rst`` for more information.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import logging
-import os
 from collections import defaultdict
 
-from mozbuild.base import MozbuildObject
-from mozbuild.util import memoize
 from slugid import nice as slugid
 
-from . import files_changed
-from .graph import Graph
-from .taskgraph import TaskGraph
-from .util.parameterization import resolve_task_references
-from .util.seta import is_low_value_task
-from .util.taskcluster import find_task_id
+from taskgraph.graph import Graph
+from taskgraph.taskgraph import TaskGraph
+from taskgraph.util.parameterization import resolve_task_references
+from taskgraph.util.python_path import import_sibling_modules
 
 logger = logging.getLogger(__name__)
 registry = {}
 
 
 def register_strategy(name, args=()):
     def wrap(cls):
         if name not in registry:
@@ -286,102 +281,14 @@ class Either(OptimizationStrategy):
             lambda sub, arg: sub.should_remove_task(task, params, arg))
 
     def should_replace_task(self, task, params, arg):
         return self._for_substrategies(
             arg,
             lambda sub, arg: sub.should_replace_task(task, params, arg))
 
 
-@register_strategy("index-search")
-class IndexSearch(OptimizationStrategy):
-
-    # A task with no dependencies remaining after optimization will be replaced
-    # if artifacts exist for the corresponding index_paths.
-    # Otherwise, we're in one of the following cases:
-    # - the task has un-optimized dependencies
-    # - the artifacts have expired
-    # - some changes altered the index_paths and new artifacts need to be
-    # created.
-    # In every of those cases, we need to run the task to create or refresh
-    # artifacts.
-
-    def should_replace_task(self, task, params, index_paths):
-        "Look for a task with one of the given index paths"
-        for index_path in index_paths:
-            try:
-                task_id = find_task_id(
-                    index_path,
-                    use_proxy=bool(os.environ.get('TASK_ID')))
-                return task_id
-            except KeyError:
-                # 404 will end up here and go on to the next index path
-                pass
-
-        return False
-
-
-@register_strategy('seta')
-class SETA(OptimizationStrategy):
-    def should_remove_task(self, task, params, _):
-        label = task.label
-
-        # we would like to return 'False, None' while it's high_value_task
-        # and we wouldn't optimize it. Otherwise, it will return 'True, None'
-        if is_low_value_task(label,
-                             params.get('project'),
-                             params.get('pushlog_id'),
-                             params.get('pushdate')):
-            # Always optimize away low-value tasks
-            return True
-        else:
-            return False
-
-
-@register_strategy("skip-unless-changed")
-class SkipUnlessChanged(OptimizationStrategy):
-    def should_remove_task(self, task, params, file_patterns):
-        # pushlog_id == -1 - this is the case when run from a cron.yml job
-        if params.get('pushlog_id') == -1:
-            return False
-
-        changed = files_changed.check(params, file_patterns)
-        if not changed:
-            logger.debug('no files found matching a pattern in `skip-unless-changed` for ' +
-                         task.label)
-            return True
-        return False
-
-
-@register_strategy("skip-unless-schedules")
-class SkipUnlessSchedules(OptimizationStrategy):
-
-    @memoize
-    def scheduled_by_push(self, repository, revision):
-        changed_files = files_changed.get_changed_files(repository, revision)
-
-        mbo = MozbuildObject.from_environment()
-        # the decision task has a sparse checkout, so, mozbuild_reader will use
-        # a MercurialRevisionFinder with revision '.', which should be the same
-        # as `revision`; in other circumstances, it will use a default reader
-        rdr = mbo.mozbuild_reader(config_mode='empty')
-
-        components = set()
-        for p, m in rdr.files_info(changed_files).items():
-            components |= set(m['SCHEDULES'].components)
-
-        return components
-
-    def should_remove_task(self, task, params, conditions):
-        if params.get('pushlog_id') == -1:
-            return False
-
-        scheduled = self.scheduled_by_push(params['head_repository'], params['head_rev'])
-        conditions = set(conditions)
-        # if *any* of the condition components are scheduled, do not optimize
-        if conditions & scheduled:
-            return False
-
-        return True
+# Trigger registration in sibling modules.
+import_sibling_modules()
 
 
 # Register composite strategies.
 register_strategy('skip-unless-schedules-or-seta', args=('skip-unless-schedules', 'seta'))(Either)
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/optimize/strategies.py
@@ -0,0 +1,110 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+import os
+
+from mozbuild.base import MozbuildObject
+from mozbuild.util import memoize
+
+from taskgraph import files_changed
+from taskgraph.optimize import register_strategy, OptimizationStrategy
+from taskgraph.util.seta import is_low_value_task
+from taskgraph.util.taskcluster import find_task_id
+
+logger = logging.getLogger(__name__)
+
+
+@register_strategy("index-search")
+class IndexSearch(OptimizationStrategy):
+
+    # A task with no dependencies remaining after optimization will be replaced
+    # if artifacts exist for the corresponding index_paths.
+    # Otherwise, we're in one of the following cases:
+    # - the task has un-optimized dependencies
+    # - the artifacts have expired
+    # - some changes altered the index_paths and new artifacts need to be
+    # created.
+    # In every of those cases, we need to run the task to create or refresh
+    # artifacts.
+
+    def should_replace_task(self, task, params, index_paths):
+        "Look for a task with one of the given index paths"
+        for index_path in index_paths:
+            try:
+                task_id = find_task_id(
+                    index_path,
+                    use_proxy=bool(os.environ.get('TASK_ID')))
+                return task_id
+            except KeyError:
+                # 404 will end up here and go on to the next index path
+                pass
+
+        return False
+
+
+@register_strategy('seta')
+class SETA(OptimizationStrategy):
+    def should_remove_task(self, task, params, _):
+        label = task.label
+
+        # we would like to return 'False, None' while it's high_value_task
+        # and we wouldn't optimize it. Otherwise, it will return 'True, None'
+        if is_low_value_task(label,
+                             params.get('project'),
+                             params.get('pushlog_id'),
+                             params.get('pushdate')):
+            # Always optimize away low-value tasks
+            return True
+        else:
+            return False
+
+
+@register_strategy("skip-unless-changed")
+class SkipUnlessChanged(OptimizationStrategy):
+    def should_remove_task(self, task, params, file_patterns):
+        # pushlog_id == -1 - this is the case when run from a cron.yml job
+        if params.get('pushlog_id') == -1:
+            return False
+
+        changed = files_changed.check(params, file_patterns)
+        if not changed:
+            logger.debug('no files found matching a pattern in `skip-unless-changed` for ' +
+                         task.label)
+            return True
+        return False
+
+
+@register_strategy("skip-unless-schedules")
+class SkipUnlessSchedules(OptimizationStrategy):
+
+    @memoize
+    def scheduled_by_push(self, repository, revision):
+        changed_files = files_changed.get_changed_files(repository, revision)
+
+        mbo = MozbuildObject.from_environment()
+        # the decision task has a sparse checkout, so, mozbuild_reader will use
+        # a MercurialRevisionFinder with revision '.', which should be the same
+        # as `revision`; in other circumstances, it will use a default reader
+        rdr = mbo.mozbuild_reader(config_mode='empty')
+
+        components = set()
+        for p, m in rdr.files_info(changed_files).items():
+            components |= set(m['SCHEDULES'].components)
+
+        return components
+
+    def should_remove_task(self, task, params, conditions):
+        if params.get('pushlog_id') == -1:
+            return False
+
+        scheduled = self.scheduled_by_push(params['head_repository'], params['head_rev'])
+        conditions = set(conditions)
+        # if *any* of the condition components are scheduled, do not optimize
+        if conditions & scheduled:
+            return False
+
+        return True
--- a/taskcluster/taskgraph/test/test_optimize.py
+++ b/taskcluster/taskgraph/test/test_optimize.py
@@ -1,40 +1,40 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import unittest
 
-from taskgraph import optimize
+from taskgraph import graph, optimize
+from taskgraph.optimize import OptimizationStrategy
 from taskgraph.taskgraph import TaskGraph
-from taskgraph import graph
 from taskgraph.task import Task
 from mozunit import main
 from slugid import nice as slugid
 
 
-class Remove(optimize.OptimizationStrategy):
+class Remove(OptimizationStrategy):
 
     def should_remove_task(self, task, params, arg):
         return True
 
 
-class Replace(optimize.OptimizationStrategy):
+class Replace(OptimizationStrategy):
 
     def should_replace_task(self, task, params, taskid):
         return taskid
 
 
 class TestOptimize(unittest.TestCase):
 
     strategies = {
-        'never': optimize.OptimizationStrategy(),
+        'never': OptimizationStrategy(),
         'remove': Remove(),
         'replace': Replace(),
     }
 
     def make_task(self, label, optimization=None, task_def=None, optimized=None,
                   task_id=None, dependencies=None):
         task_def = task_def or {'sample': 'task-def'}
         task = Task(kind='test', label=label, attributes={}, task=task_def)