Bug 1434729 - Make it possible to overwrite 'optimization' in tests transforms. r=dustin
authorMarco Castelluccio <mcastelluccio@mozilla.com>
Tue, 20 Feb 2018 19:26:32 +0100
changeset 405558 4cbab279837a9799c0c05bfd3da48a3e877f534c
parent 405557 07f09cd2e887af248f28b5c11800ba6a7efd759b
child 405559 eeb50080d6de7e10d0d38e5ae48eb43e8498f700
push id33524
push userapavel@mozilla.com
push dateTue, 27 Feb 2018 22:24:31 +0000
treeherdermozilla-central@529754159078 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1434729
milestone60.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1434729 - Make it possible to overwrite 'optimization' in tests transforms. r=dustin
taskcluster/taskgraph/transforms/task.py
taskcluster/taskgraph/transforms/tests.py
taskcluster/taskgraph/util/schema.py
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -12,22 +12,27 @@ from __future__ import absolute_import, 
 
 import hashlib
 import os
 import re
 import time
 from copy import deepcopy
 
 from mozbuild.util import memoize
-from mozbuild import schedules
 from taskgraph.util.attributes import TRUNK_PROJECTS
 from taskgraph.util.hash import hash_path
 from taskgraph.util.treeherder import split_symbol
 from taskgraph.transforms.base import TransformSequence
-from taskgraph.util.schema import validate_schema, Schema, optionally_keyed_by, resolve_keyed_by
+from taskgraph.util.schema import (
+    validate_schema,
+    Schema,
+    optionally_keyed_by,
+    resolve_keyed_by,
+    OptimizationSchema,
+)
 from taskgraph.util.scriptworker import (
     BALROG_ACTIONS,
     get_balrog_action_scope,
     get_balrog_server_scope,
     get_release_config,
 )
 from voluptuous import Any, Required, Optional, Extra
 from taskgraph import GECKO, MAX_DEPENDENCIES
@@ -226,34 +231,17 @@ task_description_schema = Schema({
     # target_task_graph regardless of filtering. Tasks included in this manner
     # will be candidates for optimization even when `optimize_target_tasks` is
     # False, unless the task was also explicitly chosen by the target_tasks
     # method.
     Required('always-target'): bool,
 
     # Optimization to perform on this task during the optimization phase.
     # Optimizations are defined in taskcluster/taskgraph/optimize.py.
-    Required('optimization'): Any(
-        # always run this task (default)
-        None,
-        # search the index for the given index namespaces, and replace this task if found
-        # the search occurs in order, with the first match winning
-        {'index-search': [basestring]},
-        # consult SETA and skip this task if it is low-value
-        {'seta': None},
-        # skip this task if none of the given file patterns match
-        {'skip-unless-changed': [basestring]},
-        # skip this task if unless the change files' SCHEDULES contains any of these components
-        {'skip-unless-schedules': list(schedules.ALL_COMPONENTS)},
-        # skip if SETA or skip-unless-schedules says to
-        {'skip-unless-schedules-or-seta': list(schedules.ALL_COMPONENTS)},
-        # only run this task if its dependencies will run (useful for follow-on tasks that
-        # are unnecessary if the parent tasks are not run)
-        {'only-if-dependencies-run': None}
-    ),
+    Required('optimization'): OptimizationSchema,
 
     # the provisioner-id/worker-type for the task.  The following parameters will
     # be substituted in this string:
     #  {level} -- the scm level of this push
     'worker-type': basestring,
 
     # Whether the job should use sccache compiler caching.
     Required('needs-sccache'): bool,
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -15,17 +15,17 @@ The test description should be fully for
 transforms, and these transforms should not embody any specific knowledge about
 what should run where. this is the wrong place for special-casing platforms,
 for example - use `all_tests.py` instead.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
-from taskgraph.util.schema import resolve_keyed_by
+from taskgraph.util.schema import resolve_keyed_by, OptimizationSchema
 from taskgraph.util.treeherder import split_symbol, join_symbol
 from taskgraph.util.platforms import platform_family
 from taskgraph.util.schema import (
     validate_schema,
     optionally_keyed_by,
     Schema,
 )
 from mozbuild.schedules import INCLUSIVE_COMPONENTS
@@ -379,16 +379,20 @@ test_description_schema = Schema({
     # the product name, defaults to firefox
     Optional('product'): basestring,
 
     # conditional files to determine when these tests should be run
     Exclusive(Optional('when'), 'optimization'): Any({
         Optional('files-changed'): [basestring],
     }),
 
+    # Optimization to perform on this task during the optimization phase.
+    # Optimizations are defined in taskcluster/taskgraph/optimize.py.
+    Exclusive(Optional('optimization'), 'optimization'): OptimizationSchema,
+
     # The SCHEDULES component for this task; this defaults to the suite
     # (not including the flavor) but can be overridden here.
     Exclusive(Optional('schedules-component'), 'optimization'): basestring,
 
     Optional('worker-type'): optionally_keyed_by(
         'test-platform',
         Any(basestring, None),
     ),
@@ -1013,16 +1017,18 @@ def make_job_description(config, tests):
             # cause it to run are annotated with SCHEDULES in moz.build,
             # so do not include the platform or any other components here
             schedules = [suite]
         else:
             schedules = [suite, platform_family(test['build-platform'])]
 
         if test.get('when'):
             jobdesc['when'] = test['when']
+        elif 'optimization' in test:
+            jobdesc['optimization'] = test['optimization']
         elif config.params['project'] != 'try' and suite not in INCLUSIVE_COMPONENTS:
             # for non-try branches and non-inclusive suites, include SETA
             jobdesc['optimization'] = {'skip-unless-schedules-or-seta': schedules}
         else:
             # otherwise just use skip-unless-schedules
             jobdesc['optimization'] = {'skip-unless-schedules': schedules}
 
         run = jobdesc['run'] = {}
--- a/taskcluster/taskgraph/util/schema.py
+++ b/taskcluster/taskgraph/util/schema.py
@@ -6,16 +6,17 @@ from __future__ import absolute_import, 
 
 import re
 import pprint
 import collections
 import voluptuous
 
 import taskgraph
 
+from mozbuild import schedules
 from .attributes import keymatch
 
 
 def validate_schema(schema, obj, msg_prefix):
     """
     Validate that object satisfies schema.  If not, generate a useful exception
     beginning with msg_prefix.
     """
@@ -197,8 +198,28 @@ def check_schema(schema):
 def Schema(*args, **kwargs):
     """
     Operates identically to voluptuous.Schema, but applying some taskgraph-specific checks
     in the process.
     """
     schema = voluptuous.Schema(*args, **kwargs)
     check_schema(schema)
     return schema
+
+
+OptimizationSchema = voluptuous.Any(
+    # always run this task (default)
+    None,
+    # search the index for the given index namespaces, and replace this task if found
+    # the search occurs in order, with the first match winning
+    {'index-search': [basestring]},
+    # consult SETA and skip this task if it is low-value
+    {'seta': None},
+    # skip this task if none of the given file patterns match
+    {'skip-unless-changed': [basestring]},
+    # skip this task if unless the change files' SCHEDULES contains any of these components
+    {'skip-unless-schedules': list(schedules.ALL_COMPONENTS)},
+    # skip if SETA or skip-unless-schedules says to
+    {'skip-unless-schedules-or-seta': list(schedules.ALL_COMPONENTS)},
+    # only run this task if its dependencies will run (useful for follow-on tasks that
+    # are unnecessary if the parent tasks are not run)
+    {'only-if-dependencies-run': None}
+)