.taskcluster.yml
author Andrew McCreight <continuation@gmail.com>
Mon, 22 Apr 2019 16:34:51 +0000
changeset 470376 3073770e06f157040f4c64951b7e8425e1ad7bbe
parent 466473 dcfe21ab348eda2fdd073fc24299b375963937f7
child 473995 80a1bc138c4f8c3b78778960769a1b4d3a3feee5
permissions -rw-r--r--
Bug 1535403 - Take indirection into account for the CC optimizations for the outer window wrapper. r=peterv Most wrapper cached C++ objects are held alive by their wrapper. The cycle collector takes advantage of this in many classes and ignores the C++ object if the wrapper is marked black. However, this is not true for the outer window's wrapper. Instead, the outer window's wrapper keeps the inner window alive. The inner window usually keeps its outer window alive, but not after it has been unlinked. For reasons I do not yet understand, the outer window's wrapper can be kept alive after the inner window it is a proxy for is unlinked. This patch fixes the cycle collector optimization for the outer window by only applying it if the outer window still has a weak reference to the inner window, which it will until the inner no longer holds the outer alive. This in turn fixes, or at least helps fix, window leaks seen intermittently when the lifetime of outer windows and docshells are tied together. The code comment is based on a review comment by peterv. Differential Revision: https://phabricator.services.mozilla.com/D27981

# This file is rendered via JSON-e by
# - mozilla-taskcluster - See
#   https://docs.taskcluster.net/reference/integrations/mozilla-taskcluster/docs/taskcluster-yml
#   {
#     tasks_for: 'hg-push',
#     push: {owner, comment, pushlog_id, pushdate},
#     repository: {url, project, level},
#     now,
#     as_slugid: // function
#     ownTaskId: // taskId of the task that will be created
#   }
#
# - cron tasks - See taskcluster/taskgraph/cron/decision.py
#   {
#     tasks_for: 'cron',
#     push: {revision, pushlog_id, pushdate, owner}
#     repository: {url, project, level},
#     cron: {task_id, job_name, job_symbol, quoted_args},
#     now,
#     ownTaskId: // taskId of the task that will be created
#   }
#
# - action tasks - See:
#   * taskcluster/taskgraph/actions/registry.py,
#   * https://docs.taskcluster.net/docs/manual/using/actions/spec
#   * ci-admin:ciadmin/generate/in_tree_actions.py
#
#   The registry generates the hookPayload that appears in actions.json, and
#   contains data from the decision task as well as JSON-e code to combine that
#   with data supplied as part of the action spec.  When the hook is fired, the
#   hookPayload is rendered with JSON-e to produce a payload for the hook task
#   template.
#
#   The ci-admin code wraps the content of this file (.taskcluster.yml) with a
#   JSON-e $let statement that produces the context described below, and
#   installs that as the hook task template.
#
#   {
#     tasks_for: 'action',
#     push: {owner, pushlog_id, revision},
#     repository: {url, project, level},
#     input,
#     parameters,
#     taskId,      // targetted taskId
#     taskGroupId, // targetted taskGroupId
#     action: {name, title, description, taskGroupId, symbol, repo_scope, cb_name}
#     ownTaskId:   // taskId of the task that will be created
#   }

version: 1
tasks:
  # NOTE: support for actions in ci-admin requires that the `tasks` property be an array *before* JSON-e rendering
  # takes place.
  - $if: 'tasks_for in ["hg-push", "action", "cron"]'
    then:
      $let:
        # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
        ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
        # ensure there's no trailing `/` on the repo URL
        repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
      in:
        taskId: {$if: 'tasks_for != "action"', then: '${ownTaskId}'}
        taskGroupId:
          $if: 'tasks_for == "action"'
          then:
            '${action.taskGroupId}'
          else:
            '${ownTaskId}' # same as taskId; this is how automation identifies a decision tsak
        schedulerId: 'gecko-level-${repository.level}'

        created: {$fromNow: ''}
        deadline: {$fromNow: '1 day'}
        expires: {$fromNow: '1 year 1 second'} # 1 second so artifacts expire first, despite rounding errors
        metadata:
          $merge:
            - owner: "${ownerEmail}"
              source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
            - $if: 'tasks_for == "hg-push"'
              then:
                name: "Gecko Decision Task"
                description: 'The task that creates all of the other tasks in the task graph'
              else:
                $if: 'tasks_for == "action"'
                then:
                  name: "Action: ${action.title}"
                  description: |
                      ${action.description}
                      
                      Action triggered by clientID `${clientId}`
                else:
                  name: "Decision Task for cron job ${cron.job_name}"
                  description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})'

        provisionerId: "aws-provisioner-v1"
        workerType: "gecko-${repository.level}-decision"

        tags:
          $if: 'tasks_for == "hg-push"'
          then:
            createdForUser: "${ownerEmail}"
            kind: decision-task
          else:
            $if: 'tasks_for == "action"'
            then:
              createdForUser: '${ownerEmail}'
              kind: 'action-callback'
            else:
              $if: 'tasks_for == "cron"'
              then:
                kind: cron-task

        routes:
          $flattenDeep:
            - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
            - $if: 'tasks_for == "hg-push"'
              then:
                - "index.gecko.v2.${repository.project}.latest.taskgraph.decision"
                - "index.gecko.v2.${repository.project}.revision.${push.revision}.taskgraph.decision"
                - "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
                - "notify.email.${ownerEmail}.on-failed"
                - "notify.email.${ownerEmail}.on-exception"
                # Send a notification email if the push comes from try
                - $if: 'repository.project == "try"'
                  then:
                    "notify.email.${ownerEmail}.on-completed"
                # These are the old index routes for the decision task.
                # They are still here so external tools that referenced them continue to work.
                - "index.gecko.v2.${repository.project}.latest.firefox.decision"
                - "index.gecko.v2.${repository.project}.revision.${push.revision}.firefox.decision"
              else:
                $if: 'tasks_for == "action"'
                then:
                - "notify.email.taskcluster-notifications+action-task@mozilla.com.on-failed"
                - "notify.email.taskcluster-notifications+action-task@mozilla.com.on-exception"
                - "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.${ownTaskId}"
                else:  # cron
                - "index.gecko.v2.${repository.project}.latest.taskgraph.decision-${cron.job_name}"
                - "index.gecko.v2.${repository.project}.revision.${push.revision}.taskgraph.decision-${cron.job_name}"
                - "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision-${cron.job_name}"
                # list each cron task on this revision, so actions can find them
                - 'index.gecko.v2.${repository.project}.revision.${push.revision}.cron.${ownTaskId}'
                # BUG 1500166 Notify ciduty by email if a nightly hook fails
                - $if: 'repository.project != "try"'
                  then:
                  - "notify.email.ciduty+failedcron@mozilla.com.on-failed"
                  - "notify.email.ciduty+exceptioncron@mozilla.com.on-exception"
                  - "notify.email.sheriffs+failedcron@mozilla.org.on-failed"
                  - "notify.email.sheriffs+exceptioncron@mozilla.org.on-exception"
                # These are the old index routes for the decision task.
                - "index.gecko.v2.${repository.project}.latest.firefox.decision-${cron.job_name}"

        scopes:
          $if: 'tasks_for == "hg-push"'
          then:
            - 'assume:repo:${repoUrl[8:]}:branch:default'
            - 'queue:route:notify.email.${ownerEmail}.*'
            - 'in-tree:hook-action:project-gecko/in-tree-action-${repository.level}-*'
          else:
            $if: 'tasks_for == "action"'
            then:
              # when all actions are hooks, we can calculate this directly rather than using a variable
              - '${action.repo_scope}'
            else:
              - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'

        dependencies: []
        requires: all-completed

        priority:
          # Most times, there is plenty of worker capacity so everything runs
          # quickly, but sometimes a storm of action tasks lands.  Then we
          # want, from highest to lowest:
          # - cron tasks (time-sensitive) (low)
          # - action tasks (avoid interfering with the other two) (very-low)
          # - decision tasks (minimize user-visible delay) (lowest)
          # SCM levels all use different workerTypes, so there is no need for priority
          # between levels; "low" is the highest priority available at all levels, and
          # nothing runs at any higher priority on these workerTypes.
          $if: "tasks_for == 'cron'"
          then: low
          else:
            $if: "tasks_for == 'action'"
            then: very-low
            else: lowest  # tasks_for == 'hg-push'
        retries: 5

        payload:
          env:
            # checkout-gecko uses these to check out the source; the inputs
            # to `mach taskgraph decision` are all on the command line.
            $merge:
              - GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
                GECKO_HEAD_REPOSITORY: '${repoUrl}'
                GECKO_HEAD_REF: '${push.revision}'
                GECKO_HEAD_REV: '${push.revision}'
                HG_STORE_PATH: /builds/worker/checkouts/hg-store
                TASKCLUSTER_CACHES: /builds/worker/checkouts
                # someday, these will be provided by the worker - Bug 1492664
                TASKCLUSTER_ROOT_URL: https://taskcluster.net
                TASKCLUSTER_PROXY_URL: http://taskcluster
              - $if: 'tasks_for == "action"'
                then:
                  ACTION_TASK_GROUP_ID: '${action.taskGroupId}'     # taskGroupId of the target task
                  ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded)
                  ACTION_INPUT: {$json: {$eval: 'input'}}
                  ACTION_CALLBACK: '${action.cb_name}'
                  ACTION_PARAMETERS: {$json: {$eval: 'parameters'}}

          cache:
            level-${repository.level}-checkouts-sparse-v2: /builds/worker/checkouts

          features:
            taskclusterProxy: true
            chainOfTrust: true

          # Note: This task is built server side without the context or tooling that
          # exist in tree so we must hard code the hash
          image: 'taskcluster/decision:2.2.0@sha256:0e9689e94605eb8395f5b49141a48148416b0d825f6f7be04c29642d1a85ee3d'

          maxRunTime: 1800

          command:
            - /builds/worker/bin/run-task
            - '--gecko-checkout=/builds/worker/checkouts/gecko'
            - '--gecko-sparse-profile=build/sparse-profiles/taskgraph'
            - '--'
            - bash
            - -cx
            - $let:
                extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
              in:
                $if: 'tasks_for == "action"'
                then: >
                  cd /builds/worker/checkouts/gecko &&
                  ln -s /builds/worker/artifacts artifacts &&
                  ./mach --log-no-times taskgraph action-callback
                else: >
                  cd /builds/worker/checkouts/gecko &&
                  ln -s /builds/worker/artifacts artifacts &&
                  ./mach --log-no-times taskgraph decision
                  --pushlog-id='${push.pushlog_id}'
                  --pushdate='${push.pushdate}'
                  --project='${repository.project}'
                  --owner='${ownerEmail}'
                  --level='${repository.level}'
                  --tasks-for='${tasks_for}'
                  --base-repository="$GECKO_BASE_REPOSITORY"
                  --head-repository="$GECKO_HEAD_REPOSITORY"
                  --head-ref="$GECKO_HEAD_REF"
                  --head-rev="$GECKO_HEAD_REV"
                  ${extraArgs}

          artifacts:
            'public':
              type: 'directory'
              path: '/builds/worker/artifacts'
              expires: {$fromNow: '1 year'}

        extra:
          $merge:
            - treeherder:
                $merge:
                  - machine:
                      platform: gecko-decision
                  - $if: 'tasks_for == "hg-push"'
                    then:
                      symbol: D
                    else:
                      $if: 'tasks_for == "action"'
                      then:
                        groupName: 'action-callback'
                        groupSymbol: AC
                        symbol: "${action.symbol}"
                      else:
                        groupSymbol: cron
                        symbol: "${cron.job_symbol}"
            - $if: 'tasks_for == "action"'
              then:
                parent: '${action.taskGroupId}'
                action:
                  name: '${action.name}'
                  context:
                    taskGroupId: '${action.taskGroupId}'
                    taskId: {$eval: 'taskId'}
                    input: {$eval: 'input'}
                    parameters: {$eval: 'parameters'}
                    clientId: {$eval: 'clientId'}
            - $if: 'tasks_for == "cron"'
              then:
                cron: {$json: {$eval: 'cron'}}
            - tasks_for: '${tasks_for}'
            # Email format for try pushes
            - $if: 'tasks_for == "hg-push" && repository.project == "try"'
              then:
                notify:
                  email:
                    subject: "Thank you for your try submission of ${push.revision}. It's the best!"
                    content: "Your try push has been submitted. It's the best! Use the link to view the status of your jobs."
                    link:
                      text: "Treeherder Jobs"
                      href: "https://treeherder.mozilla.org/#/jobs?repo=${repository.project}&revision=${push.revision}"