Bug 1450012: [taskgraph] Disable retrigger action for many tasks; r=dustin
authorTom Prince <mozilla@hocat.ca>
Tue, 16 Apr 2019 22:02:46 +0000
changeset 469730 fe831ef6691f
parent 469729 acfc18211ade
child 469731 43956eb12043
push id35880
push usercbrindusan@mozilla.com
push dateWed, 17 Apr 2019 09:36:19 +0000
treeherdermozilla-central@79e6ed0b08d6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1450012
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1450012: [taskgraph] Disable retrigger action for many tasks; r=dustin Many tasks (release tasks and cached tasks, in particular) should be re-run rather than retriggered. Disable retrigger action for those tasks by default. Differential Revision: https://phabricator.services.mozilla.com/D27206
taskcluster/ci/hazard/kind.yml
taskcluster/ci/mar-signing-autograph-stage/kind.yml
taskcluster/ci/source-test/kind.yml
taskcluster/ci/spidermonkey/kind.yml
taskcluster/ci/static-analysis-autotest/kind.yml
taskcluster/ci/test/kind.yml
taskcluster/ci/valgrind/kind.yml
taskcluster/ci/webrender/kind.yml
taskcluster/docs/attributes.rst
taskcluster/taskgraph/actions/registry.py
taskcluster/taskgraph/actions/retrigger.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/hazard/kind.yml
+++ b/taskcluster/ci/hazard/kind.yml
@@ -10,16 +10,18 @@ kind-dependencies:
 transforms:
     - taskgraph.transforms.build_attrs:transforms
     - taskgraph.transforms.build_lints:transforms
     - taskgraph.transforms.use_toolchains:transforms
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
 job-defaults:
+    attributes:
+        retrigger: true
     treeherder:
         kind: build
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 36000
         docker-image: {in-tree: debian7-amd64-build}
 
--- a/taskcluster/ci/mar-signing-autograph-stage/kind.yml
+++ b/taskcluster/ci/mar-signing-autograph-stage/kind.yml
@@ -20,8 +20,10 @@ job-template:
     treeherder-group: ms-stage
     treeherder:
         tier: 3
     description-suffix: 'autograph-stage mar signing test'
     required_signoffs:
         - mar-signing
     run-on-projects: []
     nightly: false
+    attributes:
+        retrigger: true
--- a/taskcluster/ci/source-test/kind.yml
+++ b/taskcluster/ci/source-test/kind.yml
@@ -25,14 +25,18 @@ jobs-from:
    - jsshell.yml
    - mozlint.yml
    - node.yml
    - python.yml
    - webidl.yml
    - wpt-metadata.yml
    - wpt-manifest.yml
 
+job-defaults:
+   attributes:
+      retrigger: true
+
 # This is used by run-task based tasks to lookup which build task it
 # should depend on based on its own platform.
 dependent-build-platforms:
    linux64-asan/opt: build-linux64-asan/opt
    linux64/debug: build-linux64/debug
    linux64.*: build-linux64-shippable/opt
--- a/taskcluster/ci/spidermonkey/kind.yml
+++ b/taskcluster/ci/spidermonkey/kind.yml
@@ -11,16 +11,18 @@ transforms:
     - taskgraph.transforms.spidermonkey:transforms
     - taskgraph.transforms.build_attrs:transforms
     - taskgraph.transforms.build_lints:transforms
     - taskgraph.transforms.use_toolchains:transforms
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
 job-defaults:
+    attributes:
+        retrigger: true
     treeherder:
         kind: build
         tier: 1
     index:
         product: firefox
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     run:
         using: spidermonkey
--- a/taskcluster/ci/static-analysis-autotest/kind.yml
+++ b/taskcluster/ci/static-analysis-autotest/kind.yml
@@ -10,16 +10,18 @@ kind-dependencies:
 transforms:
     - taskgraph.transforms.build_attrs:transforms
     - taskgraph.transforms.build_lints:transforms
     - taskgraph.transforms.use_toolchains:transforms
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
 job-defaults:
+    attributes:
+        retrigger: true
     index:
         product: firefox
     worker:
         skip-artifacts: true
         max-run-time: 3600
         env:
             PERFHERDER_EXTRA_OPTIONS: static-analysis-autotest
     run:
--- a/taskcluster/ci/test/kind.yml
+++ b/taskcluster/ci/test/kind.yml
@@ -30,18 +30,19 @@ jobs-from:
     - mochitest.yml
     - raptor.yml
     - raptor-chromium.yml
     - reftest.yml
     - talos.yml
     - web-platform.yml
     - xpcshell.yml
 
-
 job-defaults:
+    attributes:
+        retrigger: true
     require-signed-extensions:
         by-release-type:
             release|esr.*: true
             beta:
                 by-test-platform:
                     .*-devedition/.*: false
                     default: true
             default: false
--- a/taskcluster/ci/valgrind/kind.yml
+++ b/taskcluster/ci/valgrind/kind.yml
@@ -9,16 +9,20 @@ kind-dependencies:
 
 transforms:
     - taskgraph.transforms.build_attrs:transforms
     - taskgraph.transforms.build_lints:transforms
     - taskgraph.transforms.use_toolchains:transforms
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
+job-defaults:
+    attributes:
+        retrigger: true
+
 jobs:
     linux64-valgrind/opt:
         description: "Linux64 Valgrind Opt"
         index:
             product: firefox
             job-name: linux64-valgrind-opt
         treeherder:
             platform: linux64/opt
--- a/taskcluster/ci/webrender/kind.yml
+++ b/taskcluster/ci/webrender/kind.yml
@@ -8,16 +8,18 @@ kind-dependencies:
     - toolchain
 
 transforms:
     - taskgraph.transforms.use_toolchains:transforms
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
 job-defaults:
+    attributes:
+        retrigger: true
     run-on-projects: ['mozilla-beta', 'trunk', 'try']
     treeherder:
         tier: 1
         kind: other
     worker:
         max-run-time: 1800
         env:
             RUST_BACKTRACE: 'full'
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -302,8 +302,12 @@ openh264_rev
 Only used for openh264 plugin builds, used to signify the revision (and thus inform artifact name) of the given build.
 
 code-review
 ===========
 If a task set this boolean attribute to `true`, it will be processed by the code
 review bot, the task will ran for every new Phabricator diff.
 Any supported and detected issue will be automatically reported on the
 Phabricator revision.
+
+retrigger
+=========
+Whether the task can be retriggered, or if it needs to be re-run.
--- a/taskcluster/taskgraph/actions/registry.py
+++ b/taskcluster/taskgraph/actions/registry.py
@@ -235,16 +235,17 @@ def register_callback_action(name, title
             })
 
             return rv
 
         actions.append(Action(order, cb_name, generic, action_builder))
 
         mem['registered'] = True
         callbacks[cb_name] = cb
+        return cb
     return register_callback
 
 
 def render_actions_json(parameters, graph_config):
     """
     Render JSON object for the ``public/actions.json`` artifact.
 
     Parameters
--- a/taskcluster/taskgraph/actions/retrigger.py
+++ b/taskcluster/taskgraph/actions/retrigger.py
@@ -1,16 +1,18 @@
 # -*- coding: utf-8 -*-
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import sys
+
 import logging
 import textwrap
 
 from slugid import nice as slugid
 from .util import (
     combine_task_graph_files,
     create_tasks,
     fetch_graph_and_labels,
@@ -56,17 +58,17 @@ def retrigger_decision_action(parameters
     title='Retrigger',
     name='retrigger',
     symbol='rt',
     generic=True,
     description=(
         'Create a clone of the task.'
     ),
     order=19,  # must be greater than other orders in this file, as this is the fallback version
-    context=[{}],
+    context=[{'retrigger': 'true'}],
     schema={
         'type': 'object',
         'properties': {
             'downstream': {
                 'type': 'boolean',
                 'description': (
                     'If true, downstream tasks from this one will be cloned as well. '
                     'The dependencies will be updated to work with the new task at the root.'
@@ -79,26 +81,77 @@ def retrigger_decision_action(parameters
                 'minimum': 1,
                 'maximum': 100,
                 'title': 'Times',
                 'description': 'How many times to run each task.',
             }
         }
     }
 )
+@register_callback_action(
+    title='Retrigger (disabled)',
+    name='retrigger',
+    cb_name='retrigger-disabled',
+    symbol='rt',
+    generic=True,
+    description=(
+        'Create a clone of the task.\n\n'
+        'This type of task should typically be re-run instead of re-triggered.'
+    ),
+    order=20,  # must be greater than other orders in this file, as this is the fallback version
+    context=[{}],
+    schema={
+        'type': 'object',
+        'properties': {
+            'downstream': {
+                'type': 'boolean',
+                'description': (
+                    'If true, downstream tasks from this one will be cloned as well. '
+                    'The dependencies will be updated to work with the new task at the root.'
+                ),
+                'default': False,
+            },
+            'times': {
+                'type': 'integer',
+                'default': 1,
+                'minimum': 1,
+                'maximum': 100,
+                'title': 'Times',
+                'description': 'How many times to run each task.',
+            },
+            'force': {
+                'type': 'boolean',
+                'default': False,
+                'description': (
+                    'This task should not be re-triggered. '
+                    'This can be overridden by passing `true` here.'
+                ),
+            },
+        }
+    }
+)
 def retrigger_action(parameters, graph_config, input, task_group_id, task_id):
     decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
         parameters, graph_config)
 
     task = taskcluster.get_task_definition(task_id)
     label = task['metadata']['name']
 
     with_downstream = ' '
     to_run = [label]
 
+    if not input.get('force', None) and not full_task_graph[label].attributes.get('retrigger'):
+        logger.info(
+            "Not retriggering task {}, task should not be retrigged "
+            "and force not specified.".format(
+                label
+            )
+        )
+        sys.exit(1)
+
     if input.get('downstream'):
         to_run = full_task_graph.graph.transitive_closure(set(to_run), reverse=True).nodes
         to_run = to_run & set(label_to_taskid.keys())
         with_downstream = ' (with downstream) '
 
     times = input.get('times', 1)
     for i in xrange(times):
         create_tasks(
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -1775,20 +1775,23 @@ def build_task(config, tasks):
         if 'coalesce' in task:
             key = coalesce_key(config, task)
             routes.append('coalesce.v1.' + key)
 
         if 'priority' not in task:
             task['priority'] = get_default_priority(config.graph_config, config.params['project'])
 
         tags = task.get('tags', {})
+        attributes = task.get('attributes', {})
+
         tags.update({
             'createdForUser': config.params['owner'],
             'kind': config.kind,
             'label': task['label'],
+            'retrigger': 'true' if attributes.get('retrigger', False) else 'false'
         })
 
         task_def = {
             'provisionerId': provisioner_id,
             'workerType': worker_type,
             'routes': routes,
             'created': {'relative-datestamp': '0 seconds'},
             'deadline': {'relative-datestamp': task['deadline-after']},
@@ -1813,17 +1816,16 @@ def build_task(config, tasks):
             th_push_link = 'https://treeherder.mozilla.org/#/jobs?repo={}&revision={}'.format(
                 config.params['project'], branch_rev)
             task_def['metadata']['description'] += ' ([Treeherder push]({}))'.format(
                 th_push_link)
 
         # add the payload and adjust anything else as required (e.g., scopes)
         payload_builders[task['worker']['implementation']](config, task, task_def)
 
-        attributes = task.get('attributes', {})
         # Resolve run-on-projects
         build_platform = attributes.get('build_platform')
         resolve_keyed_by(task, 'run-on-projects', item_name=task['label'],
                          **{'build-platform': build_platform})
         attributes['run_on_projects'] = task.get('run-on-projects', ['all'])
         attributes['always_target'] = task['always-target']
         # This logic is here since downstream tasks don't always match their
         # upstream dependency's shipping_phase.