Bug 1568277 - [taskgraph] Create optimize strategy aliases for the 'test' kind r=tomprince
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 15 Aug 2019 18:48:54 +0000
changeset 488351 391a90f3f02babc4ef8a79b99b4a5a838a8b18ea
parent 488350 d2b1d6c0a7322cc546c885dc164e812b693878aa
child 488352 f8b41cbaaf8e4f5205aa3d429bd56ccd36a4ace1
push id92705
push userahalberstadt@mozilla.com
push dateThu, 15 Aug 2019 19:44:39 +0000
treeherderautoland@056d9515483c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstomprince
bugs1568277
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1568277 - [taskgraph] Create optimize strategy aliases for the 'test' kind r=tomprince This allows test tasks to declare a static optimization name, which can then be swapped in and out from the optimize code without needing to update the transforms. This will make it easier to change optimization strategies or run experimental ones. Differential Revision: https://phabricator.services.mozilla.com/D40204
taskcluster/taskgraph/optimize/__init__.py
taskcluster/taskgraph/test/test_optimize.py
taskcluster/taskgraph/transforms/tests.py
taskcluster/taskgraph/util/schema.py
--- a/taskcluster/taskgraph/optimize/__init__.py
+++ b/taskcluster/taskgraph/optimize/__init__.py
@@ -70,17 +70,20 @@ def optimize_task_graph(target_task_grap
             label_to_taskid), label_to_taskid
 
 
 def _get_optimizations(target_task_graph, strategies):
     def optimizations(label):
         task = target_task_graph.tasks[label]
         if task.optimization:
             opt_by, arg = task.optimization.items()[0]
-            return (opt_by, strategies[opt_by], arg)
+            strategy = strategies[opt_by]
+            if hasattr(strategy, 'description'):
+                opt_by += " ({})".format(strategy.description)
+            return (opt_by, strategy, arg)
         else:
             return ('never', strategies['never'], None)
     return optimizations
 
 
 def _log_optimization(verb, opt_counts):
     if opt_counts:
         logger.info(
@@ -256,16 +259,17 @@ class Either(OptimizationStrategy):
     earliest).  By default, each substrategy gets the same arg, but split_args
     can return a list of args for each strategy, if desired."""
     def __init__(self, *substrategies, **kwargs):
         missing = set(substrategies) - set(registry.keys())
         if missing:
             raise TypeError("substrategies aren't registered: {}".format(
                 ",  ".join(sorted(missing))))
 
+        self.description = "-or-".join(substrategies)
         self.substrategies = [registry[sub] for sub in substrategies]
         self.split_args = kwargs.pop('split_args', None)
         if not self.split_args:
             self.split_args = lambda arg: [arg] * len(substrategies)
         if kwargs:
             raise TypeError("unexpected keyword args")
 
     def _for_substrategies(self, arg, fn):
@@ -281,14 +285,26 @@ class Either(OptimizationStrategy):
             lambda sub, arg: sub.should_remove_task(task, params, arg))
 
     def should_replace_task(self, task, params, arg):
         return self._for_substrategies(
             arg,
             lambda sub, arg: sub.should_replace_task(task, params, arg))
 
 
+class Alias(Either):
+    """Provides an alias to an existing strategy.
+
+    This can be useful to swap strategies in and out without needing to modify
+    the task transforms.
+    """
+    def __init__(self, strategy):
+        super(Alias, self).__init__(strategy)
+
+
 # Trigger registration in sibling modules.
 import_sibling_modules()
 
 
 # Register composite strategies.
-register_strategy('skip-unless-schedules-or-seta', args=('skip-unless-schedules', 'seta'))(Either)
+register_strategy('test', args=('skip-unless-schedules', 'seta'))(Either)
+register_strategy('test-inclusive', args=('skip-unless-schedules',))(Alias)
+register_strategy('test-try', args=('skip-unless-schedules',))(Alias)
--- a/taskcluster/taskgraph/test/test_optimize.py
+++ b/taskcluster/taskgraph/test/test_optimize.py
@@ -65,16 +65,17 @@ class TestOptimize(unittest.TestCase):
             self.make_task('t1', opts.get('t1')),
             self.make_task('t2', opts.get('t2')),
             self.make_task('t3', opts.get('t3')),
             ('t3', 't2', 'dep'),
             ('t3', 't1', 'dep2'),
             ('t2', 't1', 'dep'))
 
     def assert_remove_tasks(self, graph, exp_removed, do_not_optimize=set()):
+        optimize.registry = self.strategies
         got_removed = optimize.remove_tasks(
             target_task_graph=graph,
             optimizations=optimize._get_optimizations(graph, self.strategies),
             params={},
             do_not_optimize=do_not_optimize)
         self.assertEqual(got_removed, exp_removed)
 
     def test_remove_tasks_never(self):
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -1411,25 +1411,26 @@ def make_job_description(config, tests):
             # if this is an "inclusive" test, then all files which might
             # cause it to run are annotated with SCHEDULES in moz.build,
             # so do not include the platform or any other components here
             schedules = [category]
         else:
             schedules = [category, platform_family(test['build-platform'])]
 
         if test.get('when'):
+            # This may still be used by comm-central.
             jobdesc['when'] = test['when']
         elif 'optimization' in test:
             jobdesc['optimization'] = test['optimization']
-        elif not config.params.is_try() and category not in INCLUSIVE_COMPONENTS:
-            # for non-try branches and non-inclusive suites, include SETA
-            jobdesc['optimization'] = {'skip-unless-schedules-or-seta': schedules}
+        elif config.params.is_try():
+            jobdesc['optimization'] = {'test-try': schedules}
+        elif category in INCLUSIVE_COMPONENTS:
+            jobdesc['optimization'] = {'test-inclusive': schedules}
         else:
-            # otherwise just use skip-unless-schedules
-            jobdesc['optimization'] = {'skip-unless-schedules': schedules}
+            jobdesc['optimization'] = {'test': schedules}
 
         run = jobdesc['run'] = {}
         run['using'] = 'mozharness-test'
         run['test'] = test
 
         if 'workdir' in test:
             run['workdir'] = test.pop('workdir')
 
--- a/taskcluster/taskgraph/util/schema.py
+++ b/taskcluster/taskgraph/util/schema.py
@@ -198,18 +198,20 @@ OptimizationSchema = voluptuous.Any(
     # the search occurs in order, with the first match winning
     {'index-search': [basestring]},
     # consult SETA and skip this task if it is low-value
     {'seta': None},
     # skip this task if none of the given file patterns match
     {'skip-unless-changed': [basestring]},
     # skip this task if unless the change files' SCHEDULES contains any of these components
     {'skip-unless-schedules': list(schedules.ALL_COMPONENTS)},
-    # skip if SETA or skip-unless-schedules says to
-    {'skip-unless-schedules-or-seta': list(schedules.ALL_COMPONENTS)},
+    # optimize strategy aliases for the test kind
+    {'test': list(schedules.ALL_COMPONENTS)},
+    {'test-inclusive': list(schedules.ALL_COMPONENTS)},
+    {'test-try': list(schedules.ALL_COMPONENTS)},
 )
 
 # shortcut for a string where task references are allowed
 taskref_or_string = voluptuous.Any(
     basestring,
     {voluptuous.Required('task-reference'): basestring},
     {voluptuous.Required('artifact-reference'): basestring},
 )