Bug 1450012: [taskgraph] Disable retrigger action for many tasks; r=dustin
authorTom Prince <mozilla@hocat.ca>
Tue, 16 Apr 2019 22:02:46 +0000
changeset 526531 8667f041f6c2f7adffabe2d0e65ef0dfd39c3944
parent 526530 3f746debc521a85763df5b1ab043f7849153a00a
child 526532 bb082e4ef0d4431d8f4edebd436d2063835c5873
push id2032
push userffxbld-merge
push dateMon, 13 May 2019 09:36:57 +0000
treeherdermozilla-release@455c1065dcbe [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdustin
bugs1450012
milestone67.0
Bug 1450012: [taskgraph] Disable retrigger action for many tasks; r=dustin Many tasks (release tasks and cached tasks, in particular) should be re-run rather than retriggered. Disable retrigger action for those tasks by default. Differential Revision: https://phabricator.services.mozilla.com/D27206
taskcluster/ci/hazard/kind.yml
taskcluster/ci/mar-signing-autograph-stage/kind.yml
taskcluster/ci/source-test/kind.yml
taskcluster/ci/spidermonkey/kind.yml
taskcluster/ci/static-analysis-autotest/kind.yml
taskcluster/ci/test/kind.yml
taskcluster/ci/valgrind/kind.yml
taskcluster/ci/webrender/kind.yml
taskcluster/docs/attributes.rst
taskcluster/taskgraph/actions/registry.py
taskcluster/taskgraph/actions/retrigger.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/hazard/kind.yml
+++ b/taskcluster/ci/hazard/kind.yml
@@ -10,16 +10,18 @@ kind-dependencies:
 transforms:
     - taskgraph.transforms.build_attrs:transforms
     - taskgraph.transforms.build_lints:transforms
     - taskgraph.transforms.use_toolchains:transforms
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
 job-defaults:
+    attributes:
+        retrigger: true
     treeherder:
         kind: build
         tier: 1
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         max-run-time: 36000
         docker-image: {in-tree: debian7-amd64-build}
 
--- a/taskcluster/ci/mar-signing-autograph-stage/kind.yml
+++ b/taskcluster/ci/mar-signing-autograph-stage/kind.yml
@@ -20,8 +20,10 @@ job-template:
     treeherder-group: ms-stage
     treeherder:
         tier: 3
     description-suffix: 'autograph-stage mar signing test'
     required_signoffs:
         - mar-signing
     run-on-projects: []
     nightly: false
+    attributes:
+        retrigger: true
--- a/taskcluster/ci/source-test/kind.yml
+++ b/taskcluster/ci/source-test/kind.yml
@@ -20,14 +20,18 @@ jobs-from:
    - file-metadata.yml
    - jsshell.yml
    - mozlint.yml
    - node.yml
    - python.yml
    - webidl.yml
    - wpt-manifest.yml
 
+job-defaults:
+   attributes:
+      retrigger: true
+
 # This is used by run-task based tasks to lookup which build task it
 # should depend on based on its own platform.
 dependent-build-platforms:
    linux64-asan/opt: build-linux64-asan/opt
    linux64/debug: build-linux64/debug
    linux64.*: build-linux64/pgo
--- a/taskcluster/ci/spidermonkey/kind.yml
+++ b/taskcluster/ci/spidermonkey/kind.yml
@@ -11,16 +11,18 @@ transforms:
     - taskgraph.transforms.spidermonkey:transforms
     - taskgraph.transforms.build_attrs:transforms
     - taskgraph.transforms.build_lints:transforms
     - taskgraph.transforms.use_toolchains:transforms
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
 job-defaults:
+    attributes:
+        retrigger: true
     treeherder:
         kind: build
         tier: 1
     index:
         product: firefox
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     run:
         using: spidermonkey
--- a/taskcluster/ci/static-analysis-autotest/kind.yml
+++ b/taskcluster/ci/static-analysis-autotest/kind.yml
@@ -10,16 +10,18 @@ kind-dependencies:
 transforms:
     - taskgraph.transforms.build_attrs:transforms
     - taskgraph.transforms.build_lints:transforms
     - taskgraph.transforms.use_toolchains:transforms
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
 job-defaults:
+    attributes:
+        retrigger: true
     index:
         product: firefox
     worker:
         skip-artifacts: true
         max-run-time: 3600
         env:
             PERFHERDER_EXTRA_OPTIONS: static-analysis-autotest
     run:
--- a/taskcluster/ci/test/kind.yml
+++ b/taskcluster/ci/test/kind.yml
@@ -30,18 +30,19 @@ jobs-from:
     - mochitest.yml
     - raptor.yml
     - raptor-chrome.yml
     - reftest.yml
     - talos.yml
     - web-platform.yml
     - xpcshell.yml
 
-
 job-defaults:
+    attributes:
+        retrigger: true
     require-signed-extensions:
         by-release-type:
             release|esr.*: true
             beta:
                 by-test-platform:
                     .*-devedition/.*: false
                     default: true
             default: false
--- a/taskcluster/ci/valgrind/kind.yml
+++ b/taskcluster/ci/valgrind/kind.yml
@@ -9,16 +9,20 @@ kind-dependencies:
 
 transforms:
     - taskgraph.transforms.build_attrs:transforms
     - taskgraph.transforms.build_lints:transforms
     - taskgraph.transforms.use_toolchains:transforms
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
+job-defaults:
+    attributes:
+        retrigger: true
+
 jobs:
     linux64-valgrind/opt:
         description: "Linux64 Valgrind Opt"
         index:
             product: firefox
             job-name: linux64-valgrind-opt
         treeherder:
             platform: linux64/opt
--- a/taskcluster/ci/webrender/kind.yml
+++ b/taskcluster/ci/webrender/kind.yml
@@ -8,16 +8,18 @@ kind-dependencies:
     - toolchain
 
 transforms:
     - taskgraph.transforms.use_toolchains:transforms
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
 job-defaults:
+    attributes:
+        retrigger: true
     run-on-projects: ['mozilla-beta', 'trunk', 'try']
     treeherder:
         tier: 1
         kind: other
     worker:
         max-run-time: 1800
         env:
             RUST_BACKTRACE: 'full'
--- a/taskcluster/docs/attributes.rst
+++ b/taskcluster/docs/attributes.rst
@@ -297,8 +297,12 @@ openh264_rev
 Only used for openh264 plugin builds, used to signify the revision (and thus inform artifact name) of the given build.
 
 code-review
 ===========
 If a task set this boolean attribute to `true`, it will be processed by the code
 review bot, the task will ran for every new Phabricator diff.
 Any supported and detected issue will be automatically reported on the
 Phabricator revision.
+
+retrigger
+=========
+Whether the task can be retriggered, or if it needs to be re-run.
--- a/taskcluster/taskgraph/actions/registry.py
+++ b/taskcluster/taskgraph/actions/registry.py
@@ -273,16 +273,17 @@ def register_callback_action(name, title
                 })
 
             return rv
 
         actions.append(Action(order, kind, cb_name, generic, action_builder))
 
         mem['registered'] = True
         callbacks[cb_name] = cb
+        return cb
     return register_callback
 
 
 def render_actions_json(parameters, graph_config):
     """
     Render JSON object for the ``public/actions.json`` artifact.
 
     Parameters
--- a/taskcluster/taskgraph/actions/retrigger.py
+++ b/taskcluster/taskgraph/actions/retrigger.py
@@ -1,16 +1,18 @@
 # -*- coding: utf-8 -*-
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import sys
+
 import logging
 import textwrap
 
 from slugid import nice as slugid
 from .util import (
     combine_task_graph_files,
     create_tasks,
     fetch_graph_and_labels,
@@ -56,17 +58,17 @@ def retrigger_decision_action(parameters
     title='Retrigger',
     name='retrigger',
     symbol='rt',
     generic=True,
     description=(
         'Create a clone of the task.'
     ),
     order=19,  # must be greater than other orders in this file, as this is the fallback version
-    context=[{}],
+    context=[{'retrigger': 'true'}],
     schema={
         'type': 'object',
         'properties': {
             'downstream': {
                 'type': 'boolean',
                 'description': (
                     'If true, downstream tasks from this one will be cloned as well. '
                     'The dependencies will be updated to work with the new task at the root.'
@@ -79,26 +81,77 @@ def retrigger_decision_action(parameters
                 'minimum': 1,
                 'maximum': 100,
                 'title': 'Times',
                 'description': 'How many times to run each task.',
             }
         }
     }
 )
+@register_callback_action(
+    title='Retrigger (disabled)',
+    name='retrigger',
+    cb_name='retrigger-disabled',
+    symbol='rt',
+    generic=True,
+    description=(
+        'Create a clone of the task.\n\n'
+        'This type of task should typically be re-run instead of re-triggered.'
+    ),
+    order=20,  # must be greater than other orders in this file, as this is the fallback version
+    context=[{}],
+    schema={
+        'type': 'object',
+        'properties': {
+            'downstream': {
+                'type': 'boolean',
+                'description': (
+                    'If true, downstream tasks from this one will be cloned as well. '
+                    'The dependencies will be updated to work with the new task at the root.'
+                ),
+                'default': False,
+            },
+            'times': {
+                'type': 'integer',
+                'default': 1,
+                'minimum': 1,
+                'maximum': 100,
+                'title': 'Times',
+                'description': 'How many times to run each task.',
+            },
+            'force': {
+                'type': 'boolean',
+                'default': False,
+                'description': (
+                    'This task should not be re-triggered. '
+                    'This can be overridden by passing `true` here.'
+                ),
+            },
+        }
+    }
+)
 def retrigger_action(parameters, graph_config, input, task_group_id, task_id):
     decision_task_id, full_task_graph, label_to_taskid = fetch_graph_and_labels(
         parameters, graph_config)
 
     task = taskcluster.get_task_definition(task_id)
     label = task['metadata']['name']
 
     with_downstream = ' '
     to_run = [label]
 
+    if not input.get('force', None) and not full_task_graph[label].attributes.get('retrigger'):
+        logger.info(
+            "Not retriggering task {}, task should not be retrigged "
+            "and force not specified.".format(
+                label
+            )
+        )
+        sys.exit(1)
+
     if input.get('downstream'):
         to_run = full_task_graph.graph.transitive_closure(set(to_run), reverse=True).nodes
         to_run = to_run & set(label_to_taskid.keys())
         with_downstream = ' (with downstream) '
 
     times = input.get('times', 1)
     for i in xrange(times):
         create_tasks(
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -1632,20 +1632,23 @@ def build_task(config, tasks):
         if 'coalesce' in task:
             key = coalesce_key(config, task)
             routes.append('coalesce.v1.' + key)
 
         if 'priority' not in task:
             task['priority'] = get_default_priority(config.graph_config, config.params['project'])
 
         tags = task.get('tags', {})
+        attributes = task.get('attributes', {})
+
         tags.update({
             'createdForUser': config.params['owner'],
             'kind': config.kind,
             'label': task['label'],
+            'retrigger': 'true' if attributes.get('retrigger', False) else 'false'
         })
 
         task_def = {
             'provisionerId': provisioner_id,
             'workerType': worker_type,
             'routes': routes,
             'created': {'relative-datestamp': '0 seconds'},
             'deadline': {'relative-datestamp': task['deadline-after']},
@@ -1670,17 +1673,16 @@ def build_task(config, tasks):
             th_push_link = 'https://treeherder.mozilla.org/#/jobs?repo={}&revision={}'.format(
                 config.params['project'], branch_rev)
             task_def['metadata']['description'] += ' ([Treeherder push]({}))'.format(
                 th_push_link)
 
         # add the payload and adjust anything else as required (e.g., scopes)
         payload_builders[task['worker']['implementation']](config, task, task_def)
 
-        attributes = task.get('attributes', {})
         # Resolve run-on-projects
         build_platform = attributes.get('build_platform')
         resolve_keyed_by(task, 'run-on-projects', item_name=task['label'],
                          **{'build-platform': build_platform})
         attributes['run_on_projects'] = task.get('run-on-projects', ['all'])
         attributes['always_target'] = task['always-target']
         # This logic is here since downstream tasks don't always match their
         # upstream dependency's shipping_phase.