.taskcluster.yml
author Rob Lemley <rob@thunderbird.net>
Fri, 09 Oct 2020 10:28:30 -0400
changeset 36220 02fc18d67f53e3c2b4a37a89bf568e76eb4551d9
parent 36094 e706b20531638432f9f6330a4b73731d04916e72
permissions -rw-r--r--
Bug 1669147 - [esr68] Disable release-bouncer-aliases. r=justdave a=rjl DONTBUILD This is for documentation purposes as there are no plans for further releases based on esr68. This is a recurring problem every release. The new release (eg 78.0) comes out and is followed by a release from the previous version. The download bouncer's "latest" aliases are overwritten with the old version, which is not what we want. This bug is referred to in the notes for esr78 launch so that it is not missed for Thunderbird 90. Differential Revision: https://phabricator.services.mozilla.com/D93084

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
# This file is rendered via JSON-e by
# - mozilla-taskcluster - See
#   https://docs.taskcluster.net/reference/integrations/mozilla-taskcluster/docs/taskcluster-yml
#   {
#     tasks_for: 'hg-push',
#     push: {owner, comment, pushlog_id, pushdate},
#     repository: {url, project, level},
#     now,
#     as_slugid: // function
#     ownTaskId: // taskId of the task that will be created
#   }
#
# - cron tasks - See taskcluster/taskgraph/cron/decision.py
#   {
#     tasks_for: 'cron',
#     push: {revision, pushlog_id, pushdate, owner}
#     repository: {url, project, level},
#     cron: {task_id, job_name, job_symbol, quoted_args},
#     now,
#     ownTaskId: // taskId of the task that will be created
#   }
#
# - action tasks - See:
#   * taskcluster/taskgraph/actions/registry.py,
#   * https://docs.taskcluster.net/docs/manual/using/actions/spec
#   * ci-admin:ciadmin/generate/in_tree_actions.py
#
#   The registry generates the hookPayload that appears in actions.json, and
#   contains data from the decision task as well as JSON-e code to combine that
#   with data supplied as part of the action spec.  When the hook is fired, the
#   hookPayload is rendered with JSON-e to produce a payload for the hook task
#   template.
#
#   The ci-admin code wraps the content of this file (.taskcluster.yml) with a
#   JSON-e $let statement that produces the context described below, and
#   installs that as the hook task template.
#
#   {
#     tasks_for: 'action',
#     push: {owner, pushlog_id, revision},
#     repository: {url, project, level},
#     input,
#     parameters,
#     taskId,      // targetted taskId
#     taskGroupId, // targetted taskGroupId
#     action: {name, title, description, taskGroupId, symbol, repo_scope, cb_name}
#     ownTaskId:   // taskId of the task that will be created
#     clientId:    // clientId that triggered this hook
#   }

version: 1
tasks:
    # NOTE: support for actions in ci-admin requires that the `tasks` property be an array *before* JSON-e rendering
    # takes place.
    - $if: 'tasks_for in ["hg-push", "action", "cron"]'
      then:
          $let:
              # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
              ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
              # ensure there's no trailing `/` on the repo URL
              repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
              # expire try-comm-central earlier than other branches
              expires:
                  $if: 'repository.project == "try-comm-central"'
                  then: {$fromNow: '28 days'}
                  else: {$fromNow: '1 year'}
          in:
              taskId: {$if: 'tasks_for != "action"', then: '${ownTaskId}'}
              taskGroupId:
                  $if: 'tasks_for == "action"'
                  then:
                      '${action.taskGroupId}'
                  else:
                      '${ownTaskId}'  # same as taskId; this is how automation identifies a decision tsak
              schedulerId: 'comm-level-${repository.level}'

              created: {$fromNow: ''}
              deadline: {$fromNow: '1 day'}
              expires: {$eval: 'expires'}
              metadata:
                  $merge:
                      - owner: "${ownerEmail}"
                        source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
                      - $if: 'tasks_for == "hg-push"'
                        then:
                            name: "Thunderbird Decision Task"
                            description: 'The task that creates all of the other tasks in the task graph'
                        else:
                            $if: 'tasks_for == "action"'
                            then:
                                name: "Action: ${action.title}"
                                description: |
                                        ${action.description}

                                        Action triggered by clientID `${clientId}`
                            else:
                                name: "Decision Task for cron job ${cron.job_name}"
                                description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})'

              provisionerId: "comm-${repository.level}"
              workerType: "decision"

              tags:
                  $if: 'tasks_for == "hg-push"'
                  then:
                      createdForUser: "${ownerEmail}"
                      kind: decision-task
                  else:
                      $if: 'tasks_for == "action"'
                      then:
                          createdForUser: '${ownerEmail}'
                          kind: 'action-callback'
                      else:
                          $if: 'tasks_for == "cron"'
                          then:
                              kind: cron-task

              routes:
                  $flattenDeep:
                      - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
                      - $if: 'tasks_for == "hg-push"'
                        then:
                            - "index.comm.v2.${repository.project}.latest.taskgraph.decision"
                            - "index.comm.v2.${repository.project}.revision.${push.revision}.taskgraph.decision"
                            - "index.comm.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
                            - "notify.email.${ownerEmail}.on-failed"
                            - "notify.email.${ownerEmail}.on-exception"
                        else:
                            $if: 'tasks_for == "action"'
                            then:
                                - "index.comm.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.${ownTaskId}"
                            else:  # cron
                                - "index.comm.v2.${repository.project}.latest.taskgraph.decision-${cron.job_name}"
                                - "index.comm.v2.${repository.project}.revision.${push.revision}.taskgraph.decision-${cron.job_name}"
                                - "index.comm.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision-${cron.job_name}"
                                # list each cron task on this revision, so actions can find them
                                - 'index.comm.v2.${repository.project}.revision.${push.revision}.cron.${ownTaskId}'
                                # Notify tb-builds by email if a nightly hook fails
                                - $if: 'repository.project != "try-comm-central"'
                                  then:
                                      - "notify.email.tb-builds@thunderbird.net.on-failed"
                                      - "notify.email.tb-builds@thunderbird.net.on-exception"

              scopes:
                  $if: 'tasks_for == "hg-push"'
                  then:
                      - 'assume:repo:${repoUrl[8:]}:branch:default'
                      - 'queue:route:notify.email.${ownerEmail}.*'
                      - 'in-tree:hook-action:project-comm/in-tree-action-${repository.level}-*'
                  else:
                      $if: 'tasks_for == "action"'
                      then:
                          # when all actions are hooks, we can calculate this directly rather than using a variable
                          - '${action.repo_scope}'
                      else:
                          - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'

              dependencies: []
              requires: all-completed

              priority:
                  # Most times, there is plenty of worker capacity so everything runs
                  # quickly, but sometimes a storm of action tasks lands.  Then we
                  # want, from highest to lowest:
                  # - cron tasks (time-sensitive) (low)
                  # - action tasks (avoid interfering with the other two) (very-low)
                  # - decision tasks (minimize user-visible delay) (lowest)
                  # SCM levels all use different workerTypes, so there is no need for priority
                  # between levels; "low" is the highest priority available at all levels, and
                  # nothing runs at any higher priority on these workerTypes.
                  $if: "tasks_for == 'cron'"
                  then: low
                  else:
                      $if: "tasks_for == 'action'"
                      then: very-low
                      else: lowest  # tasks_for == 'hg-push'
              retries: 5

              payload:
                  env:
                      # checkout-gecko uses these to check out the source; the inputs
                      # to `mach taskgraph decision` are all on the command line.
                      $merge:
                          - COMM_BASE_REPOSITORY: 'https://hg.mozilla.org/comm-central'
                            COMM_HEAD_REPOSITORY: '${repoUrl}'
                            COMM_HEAD_REF: '${push.revision}'
                            COMM_HEAD_REV: '${push.revision}'
                            HG_STORE_PATH: /builds/worker/checkouts/hg-store
                            TASKCLUSTER_CACHES: /builds/worker/checkouts
                            # mach generates pyc files when reading `mach_commands.py`
                            # This causes cached_task digest generation to be random for
                            # some tasks. Disable bytecode generation to work around that.
                            PYTHONDONTWRITEBYTECODE: '1'
                            # someday, these will be provided by the worker - Bug 1492664
                            TASKCLUSTER_ROOT_URL: https://taskcluster.net
                            TASKCLUSTER_PROXY_URL: http://taskcluster
                          - $if: 'tasks_for == "action"'
                            then:
                                ACTION_TASK_GROUP_ID: '${action.taskGroupId}'     # taskGroupId of the target task
                                ACTION_TASK_ID: {$json: {$eval: 'taskId'}}  # taskId of the target task (JSON-encoded)
                                ACTION_INPUT: {$json: {$eval: 'input'}}
                                ACTION_CALLBACK: '${action.cb_name}'
                                ACTION_PARAMETERS: {$json: {$eval: 'parameters'}}

                  cache:
                      comm-level-${repository.level}-checkouts-sparse-v2: /builds/worker/checkouts

                  features:
                      taskclusterProxy: true
                      chainOfTrust: true

                  # Note: This task is built server side without the context or tooling that
                  # exist in tree so we must hard code the hash
                  image: 'taskcluster/decision:2.2.0@sha256:cbeadf57300de60408bf1337e723f0cb1f0200f559799cb54deb9535d1e03b4a'

                  maxRunTime: 1800

                  command:
                      - /builds/worker/bin/comm-task-env
                      - /builds/worker/bin/run-task
                      - '--gecko-checkout=/builds/worker/checkouts/gecko'
                      - '--gecko-sparse-profile=build/sparse-profiles/taskgraph'
                      - '--comm-checkout=/builds/worker/checkouts/gecko/comm'
                      - '--'
                      - bash
                      - -cx
                      - $let:
                            extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
                        in:
                            $if: 'tasks_for == "action"'
                            then: >
                                cd /builds/worker/checkouts/gecko &&
                                ln -s /builds/worker/artifacts artifacts &&
                                ./mach --log-no-times taskgraph action-callback
                                --root=comm/taskcluster/ci
                            else: >
                                cd /builds/worker/checkouts/gecko &&
                                ln -s /builds/worker/artifacts artifacts &&
                                ./mach --log-no-times taskgraph decision
                                --root=comm/taskcluster/ci
                                --pushlog-id='${push.pushlog_id}'
                                --pushdate='${push.pushdate}'
                                --project='${repository.project}'
                                --owner='${ownerEmail}'
                                --level='${repository.level}'
                                --tasks-for='${tasks_for}'
                                --base-repository="$GECKO_BASE_REPOSITORY"
                                --head-repository="$GECKO_HEAD_REPOSITORY"
                                --head-ref="$GECKO_HEAD_REF"
                                --head-rev="$GECKO_HEAD_REV"
                                --comm-base-repository="$COMM_BASE_REPOSITORY"
                                --comm-head-repository="$COMM_HEAD_REPOSITORY"
                                --comm-head-ref="$COMM_HEAD_REF"
                                --comm-head-rev="$COMM_HEAD_REV"
                                --try-task-config-file=comm/try_task_config.json
                                ${extraArgs}

                  artifacts:
                      'public':
                          type: 'directory'
                          path: '/builds/worker/artifacts'
                          expires: {$eval: expires}

              extra:
                  $merge:
                      - treeherder:
                            $merge:
                                - machine:
                                      platform: gecko-decision
                                - $if: 'tasks_for == "hg-push"'
                                  then:
                                      symbol: D
                                  else:
                                      $if: 'tasks_for == "action"'
                                      then:
                                          groupName: 'action-callback'
                                          groupSymbol: AC
                                          symbol: "${action.symbol}"
                                      else:
                                          groupSymbol: cron
                                          symbol: "${cron.job_symbol}"
                      - $if: 'tasks_for == "action"'
                        then:
                            parent: '${action.taskGroupId}'
                            action:
                                name: '${action.name}'
                                context:
                                    taskGroupId: '${action.taskGroupId}'
                                    taskId: {$eval: 'taskId'}
                                    input: {$eval: 'input'}
                                    parameters: {$eval: 'parameters'}
                                    clientId: {$eval: 'clientId'}
                      - $if: 'tasks_for == "cron"'
                        then:
                            cron: {$json: {$eval: 'cron'}}
                      - tasks_for: '${tasks_for}'
                      # Email for all pushes should link to treeherder
                      - $if: 'tasks_for == "hg-push"'
                        then:
                            notify:
                                email:
                                    $merge:
                                        - link:
                                              text: "Treeherder Jobs"
                                              href: "https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}"