.taskcluster.yml
author Mozilla Releng Treescript <release+treescript@mozilla.org>
Mon, 22 Mar 2021 08:37:38 +0000
changeset 41418 469750840b63d92271fd91b5db4e7ec0e75c6eef
parent 40520 e6a2afef80c456633c620350cb1c899663d86264
child 42456 d6b1f17fbe822fdc8756f8282d26e999ca031d59
permissions -rw-r--r--
no bug - Bumping Thunderbird l10n changesets r=release a=l10n-bump DONTBUILD ar -> 0d19f1c07ecedb196ed5c462afe8355025d8d3c8 be -> 5c1ac7b2253fd6cc99416c5d171bb80457db5b55 bg -> 7289260eb00ffb5ebff28ace31eb1fb0a928f151 br -> a3c4e6b8ebe164b8b22be083df33f467338f324a ca -> 2bddd90cfca3259922b866c0095cddcb1e4a8cf6 cak -> 34d0c578939190cc6b08a1ea1405ab89ce8664da cs -> 7174ed1d41e8d3df28f6c9155654b31d1bd81ded cy -> 6ec17b2c9cfe627746c5e8c4d442b71ccb98511a da -> 2a0039c127a50242b19970479d06ffdd950f392b de -> fe1727679c5aa251f0b6d7ecdb55f274c48979fd dsb -> a8c9b2534c7c3a954a68e20abda112dbb0f168af el -> ddbeee1744d24b919c3d5fca8a56eff6042d871a en-CA -> cca41d76480b2b8e24bfb0946c18a9edc1e5c529 en-GB -> a57fae2878b776ee7bac53014f19585f6360475d es-AR -> ae2899c5c38f470ce3bff11df852c9dc72e71a0e es-ES -> 997fb8029ff2489ae1f826dec67a320c42ab9ae1 et -> a96e92052387c16bbcffe4cc25a942781a5060b9 eu -> 87240f51b7c640dfafdd439109e63b1f536c6086 fi -> e78ce6c8fafcd3eed25a7af6769593798154d5e0 fr -> f7df0eecd20d13d619adead8807c62ec9006389f fy-NL -> 46c116c82b236f195569f1cfcb92e1d5090ee3fe gl -> 40a0f16a7b9a184315798855ad71cef8fad6d3a1 he -> a65128376cf46c8b8ee11639a5d10ce9981699a8 hr -> 077a56b125d981426c84c9ad28e12497ef8c75cf hsb -> 70118d5baf56d1c4ecb9a91eaba48e15bef8dc1a hu -> 5e38454153738bdc3da1016a10d9435b8e856374 hy-AM -> 4cf6e3aa30586f358d74dabe872dc69f0ff893e5 id -> 727c0157707a480934ecdfae18ba991fee79e985 it -> 23cb1b9403df55510feadb7ccf6970690ce6f5ba ja -> 29ecefce0a748f46d7848b6682bede44a564a3c0 ja-JP-mac -> 5f0d96ca9de50ca47d9b1a58efe8ec8288678986 ka -> 7902ab45bc6521652763ca9117d361b55f5444ec kab -> fe2ad176fe41fb8c0b900eb46fa8920b21e66204 kk -> 15134e1e5e1248731ca56ee77da9205cfa3d60cb ko -> a72bee782fb4adc994c1ed33fa44d21c95d053a0 lt -> ebe9a3160739e04a77a9a0e3cd1fc54bf71eb2bf nb-NO -> e6367352d009f5d41c6d92d803b326c7750a91b5 nl -> c22b1ab43765214c112e87a83a3e7780695a6c70 nn-NO -> dcb52465dcc0742896c372191040b4ed67c84a0e pa-IN -> 2435f2e9153899a6a9ce714c3e2f3e185e326da1 pl -> 7563545e97325d80aba09cacc08156b3a546c04d pt-BR -> b924fc058102089db90b55512f307a5a2d31dee5 pt-PT -> 3208f934f658e1514ebfc44732b3ffc69ba7dfcb rm -> 3c8e4cd74d81aa664f17b9b2ac6982e7a3415f00 ro -> 6d580dc761bf6bfbc291fdb1f27341cffc7c9735 ru -> f663d08ee0931a5938812c638b1354514e478d94 sk -> bd2e8286afe6cb86cce9ea69487d29742dbed3b0 sl -> 5c0add6445e2ba0fa55c6b64754893d2d924fe9f sq -> 45cf8fd3abbf58a8f2da229b317a77ade02926d5 sr -> 0badca946680876e5e8588600b76df7bc777283c sv-SE -> b403b5f2c2b79ec7ddcd7df4b5f6ae7d3d066d1a th -> 16dfd654e3588349b9624c119db856ed064e4541 tr -> 0b2c1304f8be28eecc2c0a08a6ea425f9fc6ced7 uk -> 7222454c1d8a5829554db24fbc6aa636e8963f7a uz -> edb10ef52b2c242b8c96a98a5e1a7ef13f3fcd21 vi -> 8a095a6f0becde32de0023df775dadbc5be4a1ff zh-CN -> 9400573eda1af97cd311f611740cbab50279d6c4 zh-TW -> a9d71057d605be2537ed0704849e5d5c4cd44683

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

# yamllint disable rule:line-length
# This file is rendered via JSON-e by
# - hg-push - https://hg.mozilla.org/ci/ci-admin/file/default/build-decision/src/build_decision/hg_push.py
#   {
#     tasks_for: 'hg-push',
#     push: {owner, comment, pushlog_id, pushdate},
#     repository: {url, project, level},
#     now,
#     as_slugid: // function
#     ownTaskId: // taskId of the task that will be created
#   }
#
# - cron tasks - https://hg.mozilla.org/ci/ci-admin/file/default/build-decision/src/build_decision/cron/decision.py
#   {
#     tasks_for: 'cron',
#     push: {revision, pushlog_id, pushdate, owner}
#     repository: {url, project, level},
#     cron: {task_id, job_name, job_symbol, quoted_args},
#     now,
#     ownTaskId: // taskId of the task that will be created
#   }
#
# - action tasks - See:
#   * taskcluster/taskgraph/actions/registry.py,
#   * https://docs.taskcluster.net/docs/manual/using/actions/spec
#   * ci-admin:ciadmin/generate/in_tree_actions.py
#
#   The registry generates the hookPayload that appears in actions.json, and
#   contains data from the decision task as well as JSON-e code to combine that
#   with data supplied as part of the action spec.  When the hook is fired, the
#   hookPayload is rendered with JSON-e to produce a payload for the hook task
#   template.
#
#   The ci-admin code wraps the content of this file (.taskcluster.yml) with a
#   JSON-e $let statement that produces the context described below, and
#   installs that as the hook task template.
#
#   {
#     tasks_for: 'action',
#     push: {owner, pushlog_id, revision},
#     repository: {url, project, level},
#     input,
#     taskId,      // targeted taskId
#     taskGroupId, // targeted taskGroupId
#     action: {name, title, description, taskGroupId, symbol, repo_scope, cb_name}
#     ownTaskId:   // taskId of the task that will be created
#     clientId:    // clientId that triggered this hook
#   }
---
version: 1
tasks:
    # NOTE: support for actions in ci-admin requires that the `tasks` property be an array *before* JSON-e rendering
    # takes place.
    - $if: 'tasks_for in ["hg-push", "action", "cron"]'
      then:
          $let:
              # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
              ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
              # ensure there's no trailing `/` on the repo URL
              repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
              # expire try-comm-central earlier than other branches
              expires:
                  $if: 'repository.project == "try-comm-central"'
                  then: {$fromNow: '28 days'}
                  else: {$fromNow: '1 year'}
              trustDomain: comm
              treeherder_link: '[Treeherder job](https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}&selectedTaskRun=${ownTaskId})'
          in:
              taskId: {$if: 'tasks_for != "action"', then: '${ownTaskId}'}
              taskGroupId:
                  $if: 'tasks_for == "action"'
                  then:
                      '${action.taskGroupId}'
                  else:
                      '${ownTaskId}'  # same as taskId; this is how automation identifies a decision task
              schedulerId: '${trustDomain}-level-${repository.level}'

              created: {$fromNow: ''}
              deadline: {$fromNow: '1 day'}
              expires: {$eval: 'expires'}
              metadata:
                  $merge:
                      - owner: "${ownerEmail}"
                        source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
                      - $if: 'tasks_for == "hg-push"'
                        then:
                            name: "Thunderbird Decision Task"
                            description: 'The task that creates all of the other tasks in the task graph (${treeherder_link})'
                        else:
                            $if: 'tasks_for == "action"'
                            then:
                                name: "Action: ${action.title}"
                                description: |
                                    ${action.description}

                                    ${treeherder_link}

                                    Action triggered by clientID `${clientId}`
                            else:
                                name: "Decision Task for cron job ${cron.job_name}"
                                description: 'Created by a [cron task](https://firefox-ci-tc.services.mozilla.com/tasks/${cron.task_id}) (${treeherder_link})'

              provisionerId: "${trustDomain}-${repository.level}"
              workerType: "decision"

              tags:
                  $if: 'tasks_for == "hg-push"'
                  then:
                      createdForUser: "${ownerEmail}"
                      kind: decision-task
                  else:
                      $if: 'tasks_for == "action"'
                      then:
                          createdForUser: '${ownerEmail}'
                          kind: 'action-callback'
                      else:
                          $if: 'tasks_for == "cron"'
                          then:
                              kind: cron-task

              routes:
                  $flattenDeep:
                      - "tc-treeherder.v2.${repository.project}.${push.revision}"
                      - $if: 'tasks_for == "hg-push"'
                        then:
                            - "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision"
                            - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision"
                            - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
                            - "notify.email.${ownerEmail}.on-failed"
                            - "notify.email.${ownerEmail}.on-exception"
                        else:
                            $if: 'tasks_for == "action"'
                            then:
                                - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.actions.${ownTaskId}"
                                - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.${ownTaskId}"
                            else:  # cron
                                - "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision-${cron.job_name}"
                                - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision-${cron.job_name}"
                                - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision-${cron.job_name}"
                                # list each cron task on this revision, so actions can find them
                                - 'index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.cron.${ownTaskId}'
                                # Notify tb-builds by email if a nightly hook fails
                                - $if: 'repository.project != "try-comm-central"'
                                  then:
                                      - "notify.email.tb-builds@thunderbird.net.on-failed"
                                      - "notify.email.tb-builds@thunderbird.net.on-exception"

              scopes:
                  $if: 'tasks_for == "hg-push"'
                  then:
                      - 'assume:repo:${repoUrl[8:]}:branch:default'
                      - 'queue:route:notify.email.${ownerEmail}.*'
                      - 'in-tree:hook-action:project-${trustDomain}/in-tree-action-${repository.level}-*'
                      - 'index:insert-task:${trustDomain}.v2.${repository.project}.*'
                  else:
                      $if: 'tasks_for == "action"'
                      then:
                          # when all actions are hooks, we can calculate this directly rather than using a variable
                          - '${action.repo_scope}'
                      else:
                          - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'

              dependencies: []
              requires: all-completed

              priority:
                  # Most times, there is plenty of worker capacity so everything runs
                  # quickly, but sometimes a storm of action tasks lands.  Then we
                  # want, from highest to lowest:
                  # - cron tasks (time-sensitive) (low)
                  # - action tasks (avoid interfering with the other two) (very-low)
                  # - decision tasks (minimize user-visible delay) (lowest)
                  # SCM levels all use different workerTypes, so there is no need for priority
                  # between levels; "low" is the highest priority available at all levels, and
                  # nothing runs at any higher priority on these workerTypes.
                  $if: "tasks_for == 'cron'"
                  then: low
                  else:
                      $if: "tasks_for == 'action'"
                      then: very-low
                      else: lowest  # tasks_for == 'hg-push'
              retries:
                  $if: "tasks_for == 'hg-push'"
                  then: 0
                  else: 5

              payload:
                  env:
                      # run-task uses these to check out the source; the inputs
                      # to `mach taskgraph decision` are all on the command line.
                      $merge:
                          - COMM_BASE_REPOSITORY: 'https://hg.mozilla.org/comm-central'
                            COMM_HEAD_REPOSITORY: '${repoUrl}'
                            COMM_HEAD_REF: '${push.revision}'
                            COMM_HEAD_REV: '${push.revision}'
                            HG_STORE_PATH: /builds/worker/checkouts/hg-store
                            TASKCLUSTER_CACHES: /builds/worker/checkouts
                            MOZ_AUTOMATION: '1'
                            # mach generates pyc files when reading `mach_commands.py`
                            # This causes cached_task digest generation to be random for
                            # some tasks. Disable bytecode generation to work around that.
                            PYTHONDONTWRITEBYTECODE: '1'
                          - $if: 'tasks_for == "action"'
                            then:
                                ACTION_TASK_GROUP_ID: '${action.taskGroupId}'  # taskGroupId of the target task
                                ACTION_TASK_ID: {$json: {$eval: 'taskId'}}  # taskId of the target task (JSON-encoded)
                                ACTION_INPUT: {$json: {$eval: 'input'}}
                                ACTION_CALLBACK: '${action.cb_name}'

                  cache:
                      "${trustDomain}-level-${repository.level}-checkouts-sparse-v2": /builds/worker/checkouts

                  features:
                      taskclusterProxy: true
                      chainOfTrust: true

                  # Note: This task is built server side without the context or tooling that
                  # exist in tree so we must hard code the hash
                  image: 'taskcluster/decision:2.2.0@sha256:cbeadf57300de60408bf1337e723f0cb1f0200f559799cb54deb9535d1e03b4a'

                  maxRunTime: 1800

                  command:
                      - /builds/worker/bin/comm-task-env
                      - /builds/worker/bin/run-task
                      - '--gecko-checkout=/builds/worker/checkouts/gecko'
                      - '--gecko-sparse-profile=build/sparse-profiles/taskgraph'
                      - '--comm-checkout=/builds/worker/checkouts/gecko/comm'
                      - '--'
                      - bash
                      - -cx
                      - $let:
                            extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
                        in:
                            $if: 'tasks_for == "action"'
                            then: >
                                cd /builds/worker/checkouts/gecko &&
                                ln -s /builds/worker/artifacts artifacts &&
                                ./mach --log-no-times taskgraph action-callback
                                --root=comm/taskcluster/ci
                            else: >
                                cd /builds/worker/checkouts/gecko &&
                                ln -s /builds/worker/artifacts artifacts &&
                                ./mach --log-no-times taskgraph decision
                                --root=comm/taskcluster/ci
                                --pushlog-id='${push.pushlog_id}'
                                --pushdate='${push.pushdate}'
                                --project='${repository.project}'
                                --owner='${ownerEmail}'
                                --level='${repository.level}'
                                --tasks-for='${tasks_for}'
                                --base-repository="$GECKO_BASE_REPOSITORY"
                                --head-repository="$GECKO_HEAD_REPOSITORY"
                                --head-ref="$GECKO_HEAD_REF"
                                --head-rev="$GECKO_HEAD_REV"
                                --comm-base-repository="$COMM_BASE_REPOSITORY"
                                --comm-head-repository="$COMM_HEAD_REPOSITORY"
                                --comm-head-ref="$COMM_HEAD_REF"
                                --comm-head-rev="$COMM_HEAD_REV"
                                --try-task-config-file=comm/try_task_config.json
                                ${extraArgs}

                  artifacts:
                      'public':
                          type: 'directory'
                          path: '/builds/worker/artifacts'
                          expires: {$eval: expires}
                      'public/docker-contexts':
                          type: 'directory'
                          path: '/builds/worker/checkouts/gecko/docker-contexts'
                          # This needs to be at least the deadline of the
                          # decision task + the docker-image task deadlines.
                          # It is set to a week to allow for some time for
                          # debugging, but they are not useful long-term.
                          expires: {$fromNow: '7 day'}

              extra:
                  $merge:
                      - treeherder:
                            $merge:
                                - machine:
                                      platform: gecko-decision
                                - $if: 'tasks_for == "hg-push"'
                                  then:
                                      symbol: D
                                  else:
                                      $if: 'tasks_for == "action"'
                                      then:
                                          groupName: 'action-callback'
                                          groupSymbol: AC
                                          symbol: "${action.symbol}"
                                      else:
                                          groupSymbol: cron
                                          symbol: "${cron.job_symbol}"
                      - $if: 'tasks_for == "action"'
                        then:
                            parent: '${action.taskGroupId}'
                            action:
                                name: '${action.name}'
                                context:
                                    taskGroupId: '${action.taskGroupId}'
                                    taskId: {$eval: 'taskId'}
                                    input: {$eval: 'input'}
                                    clientId: {$eval: 'clientId'}
                      - $if: 'tasks_for == "cron"'
                        then:
                            cron: {$json: {$eval: 'cron'}}
                      - tasks_for: '${tasks_for}'
                      # Email for all pushes should link to treeherder
                      - $if: 'tasks_for == "hg-push"'
                        then:
                            notify:
                                email:
                                    $merge:
                                        - link:
                                              text: "Treeherder Jobs"
                                              href: "https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}"