author Rob Lemley <rob@thunderbird.net>
Mon, 26 Aug 2019 21:20:54 -0400
changeset 35928 0bd1a3c3bee51d0643e1f3e9511876d27df182c1
parent 32976 73984916cb117ed8cb992a9c33aafb246c9f6a6f
child 35950 8deffa88139ba19c7f30254eb202ddca2bd8c9fd
permissions -rw-r--r--
Bug 1507754 - Check source repositories and changesets during configure. r=darktrojan a=jorgk PACKAGERS: if you update application.ini, platform.ini, or source-repo.h in some way during your process, you might need to change something. Make sure that the source repositories for both Mozilla and Comm can be found during mach configure and abort if they cannot. For Taskcluster builds, there are various environment variables that can be relied upon. Local builds present a challenge. Chances are those variables are not set. I came up with a set of checks and keep trying until something works. For comm-* code: - Look for MOZ_SOURCE_REPO and MOZ_SOURCE_CHANGESET environment vars. This is counter-intuitive, but it's the current status-quo for Taskcluster builds. Those variables are set to the comm values. - Next, try use the Mercurial source checkout itself. Uses the same technique as Mozilla code does in build/variables.py. - Last, try to use a file named "sourcestamp.txt". That file is part of our source tar files that get built for releases. - Finally, if those MOZ_SOURCE environment variables were not set, set them. This is needed because old-configure will look for them and set buildconfig variables with them when it runs later during the configure process. - Additionally, set MOZ_COMM_SOURCE_REPO and MOZ_COMM_SOURCE_CHANGESET in buildconfig. Code in the comm- tree should prefer those values over the generic MOZ_SOURCE_* values that the Mozilla code will look at. For the Gecko/Mozilla source repository information, it's almost the same process. - Check for GECKO_SOURCE_REPO and GECKO_SOURCE_REV environment variables first. Taskcluster sets these based on comm/.gecko_rev.yml. - Next, try comm/.gecko_rev.yml itself. PyYAML is not required as the file is pretty simple to parse. Release builds are pinned to a specific revision hash, so we can use that. Builds from comm-central pin to "default" though, so next try running "hg id" in $topsrcdir to get the revision hash. - If for some reason there's no .gecko_rev.yml and it's not a Mercurial checkout, try the sourcestamp.txt file. - Set MOZ_GECKO_SOURCE_REPO and MOZ_GECKO_SOURCE_CHANGESET in buildconfig. mach configure should fail if any one of those values cannot be determined. The error message will suggest setting the environment variables; ideally that is not necessary.

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# yamllint disable rule:indentation
# This file is rendered via JSON-e by
# - mozilla-taskcluster - See
#   https://docs.taskcluster.net/reference/integrations/mozilla-taskcluster/docs/taskcluster-yml
#   {
#     tasks_for: 'hg-push',
#     push: {owner, comment, pushlog_id, pushdate},
#     repository: {url, project, level},
#     now,
#     as_slugid: // function
#     ownTaskId: // taskId of the task that will be created
#   }
# - cron tasks - See taskcluster/taskgraph/cron/decision.py
#   {
#     tasks_for: 'cron',
#     push: {revision, pushlog_id, pushdate, owner}
#     repository: {url, project, level},
#     cron: {task_id, job_name, job_symbol, quoted_args},
#     now,
#     ownTaskId: // taskId of the task that will be created
#   }
# - action tasks - See:
#   * taskcluster/taskgraph/actions/registry.py,
#   * https://docs.taskcluster.net/docs/manual/using/actions/spec
#   * ci-admin:ciadmin/generate/in_tree_actions.py
#   The registry generates the hookPayload that appears in actions.json, and
#   contains data from the decision task as well as JSON-e code to combine that
#   with data supplied as part of the action spec.  When the hook is fired, the
#   hookPayload is rendered with JSON-e to produce a payload for the hook task
#   template.
#   The ci-admin code wraps the content of this file (.taskcluster.yml) with a
#   JSON-e $let statement that produces the context described below, and
#   installs that as the hook task template.
#   {
#     tasks_for: 'action',
#     push: {owner, pushlog_id, revision},
#     repository: {url, project, level},
#     input,
#     parameters,
#     taskId,      // targetted taskId
#     taskGroupId, // targetted taskGroupId
#     action: {name, title, description, taskGroupId, symbol, repo_scope, cb_name}
#     ownTaskId:   // taskId of the task that will be created
#   }

version: 1

  - $let:
      # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
      ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
      # ensure there's no trailing `/` on the repo URL
      repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
      # Hardcode cron push info for now, so that we can transition to using real values without breaking callers of Chain of Trust
      _pushId: {$if: 'tasks_for == "cron"', then: '-1', else: {$eval: 'push.pushlog_id'}}
      taskId: {$if: 'tasks_for != "action"', then: '${ownTaskId}'}
        $if: 'tasks_for == "action"'
          '${ownTaskId}'  # same as taskId; this is how automation identifies a decision tsak
      schedulerId: 'comm-level-${repository.level}'

      created: {$fromNow: ''}
      deadline: {$fromNow: '1 day'}
      expires: {$fromNow: '1 year 1 second'}  # 1 second so artifacts expire first, despite rounding errors

          - owner: "${ownerEmail}"
            source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
          - $if: 'tasks_for == "hg-push"'
              name: "Thunderbird Decision Task"
              description: 'The task that creates all of the other tasks in the task graph'
              $if: 'tasks_for == "action"'
                name: "Action: ${action.title}"
                description: '${action.description}'
                name: "Decision Task for cron job ${cron.job_name}"
                description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})'

      provisionerId: "aws-provisioner-v1"
      workerType: "gecko-${repository.level}-decision"

        $if: 'tasks_for == "hg-push"'
        then: {createdForUser: "${ownerEmail}"}
          $if: 'tasks_for == "action"'
            createdForUser: '${ownerEmail}'
            kind: 'action-callback'

          - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
          - "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
          - $if: 'tasks_for == "hg-push"'
              - "index.comm.v2.${repository.project}.latest.taskgraph.decision"
              - "index.comm.v2.${repository.project}.revision.${push.revision}.taskgraph.decision"
              - "index.comm.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
              - "notify.email.${ownerEmail}.on-failed"
              - "notify.email.${ownerEmail}.on-exception"
              $if: 'tasks_for == "action"'
                - "notify.email.taskcluster-notifications+action-task@mozilla.com.on-failed"
                - "notify.email.taskcluster-notifications+action-task@mozilla.com.on-exception"
                - "index.comm.v2.${repository.project}.pushlog-id.${_pushId}.actions.${ownTaskId}"
                - "index.comm.v2.${repository.project}.latest.taskgraph.decision-${cron.job_name}"

        $if: 'tasks_for == "hg-push"'
          - 'assume:repo:${repoUrl[8:]}:branch:default'
          - 'queue:route:notify.email.${ownerEmail}.*'
          - 'in-tree:hook-action:project-comm/in-tree-action-${repository.level}-*'
          $if: 'tasks_for == "action"'
            - '${action.repo_scope}'
            - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'

      dependencies: []
      requires: all-completed

      priority: lowest
      retries: 5

          # checkout-gecko uses these to check out the source; the inputs
          # to `mach taskgraph decision` are all on the command line.
            - COMM_BASE_REPOSITORY: 'https://hg.mozilla.org/comm-central'
              COMM_HEAD_REPOSITORY: '${repoUrl}'
              COMM_HEAD_REF: '${push.revision}'
              COMM_HEAD_REV: '${push.revision}'
              HG_STORE_PATH: /builds/worker/checkouts/hg-store
              TASKCLUSTER_CACHES: /builds/worker/checkouts
              # someday, these will be provided by the worker - Bug 1492664
              TASKCLUSTER_ROOT_URL: https://taskcluster.net
              TASKCLUSTER_PROXY_URL: http://taskcluster
            - $if: 'tasks_for == "action"'
                ACTION_TASK_GROUP_ID: '${action.taskGroupId}'  # taskGroupId of the target task
                ACTION_TASK_ID: {$json: {$eval: 'taskId'}}     # taskId of the target task (JSON-encoded)
                ACTION_INPUT: {$json: {$eval: 'input'}}
                ACTION_CALLBACK: '${action.cb_name}'
                ACTION_PARAMETERS: {$json: {$eval: 'parameters'}}

          level-${repository.level}-checkouts-sparse-v2: /builds/worker/checkouts

          taskclusterProxy: true
          chainOfTrust: true

        # Note: This task is built server side without the context or tooling that
        # exist in tree so we must hard code the hash
        image: 'taskcluster/decision:2.2.0@sha256:cbeadf57300de60408bf1337e723f0cb1f0200f559799cb54deb9535d1e03b4a'

        maxRunTime: 1800

          - /builds/worker/bin/comm-task-env
          - /builds/worker/bin/run-task
          - '--gecko-checkout=/builds/worker/checkouts/gecko'
          - '--gecko-sparse-profile=build/sparse-profiles/taskgraph'
          - '--comm-checkout=/builds/worker/checkouts/gecko/comm'
          - '--'
          - bash
          - -cx
          - $let:
              extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
              $if: 'tasks_for == "action"'
              then: >
                cd /builds/worker/checkouts/gecko &&
                ln -s /builds/worker/artifacts artifacts &&
                ./mach --log-no-times taskgraph action-callback
              else: >
                cd /builds/worker/checkouts/gecko &&
                ln -s /builds/worker/artifacts artifacts &&
                ./mach --log-no-times taskgraph decision

            type: 'directory'
            path: '/builds/worker/artifacts'
            expires: {$fromNow: '1 year'}

          - treeherder:
                - machine:
                    platform: gecko-decision
                - $if: 'tasks_for == "hg-push"'
                    symbol: D
                    $if: 'tasks_for == "action"'
                      groupName: 'action-callback'
                      groupSymbol: AC
                      symbol: "${action.symbol}"
                      groupSymbol: cron
                      symbol: "${cron.job_symbol}"
          - $if: 'tasks_for == "action"'
              parent: '${action.taskGroupId}'
                name: '${action.name}'
                  taskGroupId: '${action.taskGroupId}'
                  taskId: {$eval: 'taskId'}
                  input: {$eval: 'input'}
                  parameters: {$eval: 'parameters'}
          - $if: 'tasks_for == "cron"'
              cron: {$json: {$eval: 'cron'}}
          - tasks_for: '${tasks_for}'