.taskcluster.yml
author Aki Sasaki <asasaki@mozilla.com>
Tue, 30 Mar 2021 18:22:06 +0000
changeset 275 949eacedadb887cc3ec16b6d42c5217504e9fb40
parent 262 d207a5df41d695ea7f599ae5a1b3cfeed4d9c11a
permissions -rw-r--r--
Bug 1663474 - add github token secret to github cron hooks. r=releng-reviewers,mtabara Also add another cron commandline option if this is a github repo. Differential Revision: https://phabricator.services.mozilla.com/D110188

# This file is rendered via JSON-e by
# - mozilla-taskcluster - https://docs.taskcluster.net/reference/integrations/mozilla-taskcluster/docs/taskcluster-yml
# - cron tasks - taskcluster/taskgraph/cron/decision.py
# - action tasks - taskcluster/taskgraph/actions/registry.py
---
version: 1
tasks:
    # NOTE: support for actions in ci-admin requires that the `tasks` property be an array *before* JSON-e rendering
    # takes place.
    - $let:
          taskgraph:
              branch: taskgraph
              revision: 12992b0f984884ec2b0a7bdedc3b3ba467363eb4
          trustDomain: ci
      in:
          $if: 'tasks_for in ["hg-push", "action", "cron"]'
          then:
              $let:
                  # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
                  ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
                  # ensure there's no trailing `/` on the repo URL
                  repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
              in:
                  taskId: {$if: 'tasks_for != "action"', then: '${ownTaskId}'}
                  taskGroupId:
                      $if: 'tasks_for == "action"'
                      then:
                          '${action.taskGroupId}'
                      else:
                          '${ownTaskId}'  # same as taskId; this is how automation identifies a decision task
                  schedulerId: '${trustDomain}-level-${repository.level}'

                  created: {$fromNow: ''}
                  deadline: {$fromNow: '1 day'}
                  expires: {$fromNow: '1 year 1 second'}  # 1 second so artifacts expire first, despite rounding errors
                  metadata:
                      $merge:
                          - owner: "${ownerEmail}"
                            source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
                          - $if: 'tasks_for == "hg-push"'
                            then:
                                name: "Decision Task"
                                description: 'The task that creates all of the other tasks in the task graph'
                            else:
                                $if: 'tasks_for == "action"'
                                then:
                                    name: "Action: ${action.title}"
                                    description: '${action.description}'
                                else:
                                    name: "Decision Task for cron job ${cron.job_name}"
                                    description: 'Created by a [cron task](https://firefox-ci-tc.services.mozilla.com/tasks/${cron.task_id})'
                  provisionerId: "${trustDomain}-${repository.level}"
                  workerType: "decision"
                  tags:
                      $if: 'tasks_for == "hg-push"'
                      then:
                          createdForUser: "${ownerEmail}"
                          kind: decision-task
                      else:
                          $if: 'tasks_for == "action"'
                          then:
                              createdForUser: '${ownerEmail}'
                              kind: 'action-callback'
                          else:
                              $if: 'tasks_for == "cron"'
                              then:
                                  kind: cron-task

                  routes:
                      $flatten:
                          - "tc-treeherder.v2.${repository.project}.${push.revision}"
                          - $if: 'tasks_for == "hg-push"'
                            then:
                                - "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision"
                                - "index.${trustDomain}.v2.${repository.project}.revision.${push.revision}.taskgraph.decision"
                                - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
                            else:
                                $if: 'tasks_for == "action"'
                                then:
                                    - "index.${trustDomain}.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.${ownTaskId}"
                                else:  # cron
                                    - "index.${trustDomain}.v2.${repository.project}.latest.taskgraph.decision-${cron.job_name}"

                  scopes:
                      $if: 'tasks_for == "hg-push"'
                      then:
                          - 'assume:repo:${repoUrl[8:]}:branch:default'
                      else:
                          $if: 'tasks_for == "action"'
                          then:
                              - 'assume:repo:${repoUrl[8:]}:action:${action.action_perm}'
                          else:
                              - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'

                  dependencies: []
                  requires: all-completed

                  priority:
                      # Most times, there is plenty of worker capacity so everything runs
                      # quickly, but sometimes a storm of action tasks lands.  Then we
                      # want, from highest to lowest:
                      # - cron tasks (time-sensitive) (low)
                      # - decision tasks (minimize user-visible delay) (very-low)
                      # - action tasks (avoid interfering with the other two) (lowest)
                      # SCM levels all use different workerTypes, so there is no need for priority
                      # between levels; "low" is the highest priority available at all levels, and
                      # nothing runs at any higher priority on these workerTypes.
                      $if: "tasks_for == 'cron'"
                      then: low
                      else:
                          $if: "tasks_for == 'hg-push'"
                          then: very-low
                          else: lowest  # tasks_for == 'action'
                  retries: 5

                  payload:
                      env:
                          # run-task uses these to check out the source; the inputs
                          # to `mach taskgraph decision` are all on the command line.
                          $merge:
                              - $if: 'repository.project[:8] == "ci-admin"'
                                then:
                                    ADMIN_BASE_REPOSITORY: '${repoUrl}'
                                    ADMIN_HEAD_REPOSITORY: '${repoUrl}'
                                    ADMIN_HEAD_REF: '${push.revision}'
                                    ADMIN_HEAD_REV: '${push.revision}'
                                    ADMIN_REPOSITORY_TYPE: hg
                                else:
                                    CONFIG_BASE_REPOSITORY: '${repoUrl}'
                                    CONFIG_HEAD_REPOSITORY: '${repoUrl}'
                                    CONFIG_HEAD_REF: '${push.revision}'
                                    CONFIG_HEAD_REV: '${push.revision}'
                                    CONFIG_REPOSITORY_TYPE: hg
                                    ADMIN_BASE_REPOSITORY: 'https://hg.mozilla.org/ci/ci-admin'
                                    ADMIN_HEAD_REPOSITORY: 'https://hg.mozilla.org/ci/ci-admin'
                                    ADMIN_HEAD_REF: 'default'
                                    ADMIN_REPOSITORY_TYPE: hg
                                    TRY_TASK_CONFIG_FILE: 'config/try_task_config.json'
                              - TASKGRAPH_BASE_REPOSITORY: https://hg.mozilla.org/ci/taskgraph
                                TASKGRAPH_HEAD_REPOSITORY: https://hg.mozilla.org/ci/${taskgraph.branch}
                                TASKGRAPH_HEAD_REV: ${taskgraph.revision}
                                TASKGRAPH_REPOSITORY_TYPE: hg
                                REPOSITORIES: {$json: {admin: ci-admin, config: ci-config, taskgraph: "Taskgraph"}}
                                HG_STORE_PATH: /builds/worker/checkouts/hg-store
                              - $if: 'tasks_for == "action"'
                                then:
                                    ACTION_TASK_GROUP_ID: '${action.taskGroupId}'  # taskGroupId of the target task
                                    ACTION_TASK_ID: {$json: {$eval: 'taskId'}}  # taskId of the target task (JSON-encoded)
                                    ACTION_INPUT: {$json: {$eval: 'input'}}
                                    ACTION_CALLBACK: '${action.cb_name}'

                      cache:
                          ${trustDomain}-level-${repository.level}-checkouts-sparse-v2: /builds/worker/checkouts

                      features:
                          taskclusterProxy: true
                          chainOfTrust: true

                      # Note: This task is built server side without the context or tooling that
                      # exist in tree so we must hard code the hash
                      image:
                          mozillareleases/taskgraph:decision-21bef1bc0f11e62c7a23384584f9f8f0d96e95eef192e5bb599fc82ba55c81a7@sha256:53dcd47f72a09c12dd528812b2c4031bbb7db9455959747eb059e5e590f00487

                      maxRunTime: 1800

                      command:
                          $flatten:
                              - /usr/local/bin/run-task
                              - '--admin-checkout=/builds/worker/checkouts/src'
                              - '--taskgraph-checkout=/builds/worker/checkouts/taskgraph'
                              - $if: 'repository.project[:16] == "ci-configuration"'
                                then:
                                    - '--config-checkout=/builds/worker/checkouts/src/config'
                              - '--task-cwd=/builds/worker/checkouts/src'
                              - '--'
                              - bash
                              - -cx
                              - $let:
                                    extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
                                in:
                                    $if: 'tasks_for == "action"'
                                    then: >
                                        cd /builds/worker/checkouts/src &&
                                        ln -s /builds/worker/artifacts artifacts &&
                                        taskgraph action-callback
                                    else: >
                                        pip install --user /builds/worker/checkouts/taskgraph &&
                                        ln -s /builds/worker/artifacts artifacts &&
                                        ~/.local/bin/taskgraph decision
                                        --pushlog-id='${push.pushlog_id}'
                                        --pushdate='${push.pushdate}'
                                        --project='${repository.project}'
                                        --message=""
                                        --owner='${ownerEmail}'
                                        --level='${repository.level}'
                                        --base-repository="${repoUrl}"
                                        --head-repository="${repoUrl}"
                                        --head-ref="${push.revision}"
                                        --head-rev="${push.revision}"
                                        --repository-type=hg
                                        --tasks-for='${tasks_for}'
                                        $${TRY_TASK_CONFIG_FILE+--try-task-config-file="$${TRY_TASK_CONFIG_FILE}"}
                                        ${extraArgs}

                      artifacts:
                          'public':
                              type: 'directory'
                              path: '/builds/worker/artifacts'
                              expires: {$fromNow: '1 year'}
                          'public/docker-contexts':
                              type: 'directory'
                              path: '/builds/worker/checkouts/src/docker-contexts'
                              # This needs to be at least the deadline of the
                              # decision task + the docker-image task deadlines.
                              # It is set to a week to allow for some time for
                              # debugging, but they are not useful long-term.
                              expires: {$fromNow: '7 day'}

                  extra:
                      $merge:
                          - treeherder:
                                $merge:
                                    - machine:
                                          platform: gecko-decision
                                    - $if: 'tasks_for == "hg-push"'
                                      then:
                                          symbol: D
                                      else:
                                          $if: 'tasks_for == "action"'
                                          then:
                                              groupName: 'action-callback'
                                              groupSymbol: AC
                                              symbol: "${action.symbol}"
                                          else:
                                              groupSymbol: cron
                                              symbol: "${cron.job_symbol}"
                          - $if: 'tasks_for == "action"'
                            then:
                                parent: '${action.taskGroupId}'
                                action:
                                    name: '${action.name}'
                                    context:
                                        taskGroupId: '${action.taskGroupId}'
                                        taskId: {$eval: 'taskId'}
                                        input: {$eval: 'input'}
                          - $if: 'tasks_for == "cron"'
                            then:
                                cron: {$json: {$eval: 'cron'}}
                          - tasks_for: '${tasks_for}'