author Mike Shal <mshal@mozilla.com>
Thu, 24 Aug 2017 22:52:01 -0400
changeset 434519 cb773c661e0ca0bf297e977343076bef34411523
parent 429607 bda524beac249b64aa36016800502a34073bf35a
child 437495 215bc8a3310cde31dd6f09c72f8e5c84a904023f
permissions -rw-r--r--
Bug 1402012 - Update buildconfig.py to use PartialConfigEnvironment; r=glandium By using the PartialConfigEnvironment, the clients of buildconfig will depend on config.statusd/ files instead of config.status directly. Clients can access substs and defines using buildconfig.substs['FOO'] or buildconfig.defines['BAR'], and then collect file-level dependencies for make using buildconfig.get_dependencies(). All GENERATED_FILES rules already make use of this because file_generate.py automatically includes these dependencies (along with all python modules loaded). As a result of this commit, re-running configure will no longer cause the world to be rebuilt. Although config.status is updated, no build steps use config.status directly and instead depend on values in config.statusd/, which are written with FileAvoidWrite. Since those files are not official targets according to the make backend, make won't try to continually rebuild the backend when those files are out of date. And since they are FileAvoidWrite, make will only re-run dependent steps if the actual configure value has changed. As a result of using JSON to load data from the config.statusd directory, substs can be unicode (instead of a bare string type). generate_certdata.py converts the subst manually to a string so the value can be exported to the environment without issue on Windows. Additionally, patching the buildconfig.substs dict no longer works, so the unit-symbolstore.py test was modified to patch the underlying buildconfig.substs._dict instead. The other files that needed to be modified make use of all the defines for the preprocessor. Those that are used during 'mach build' now use buildconfig.defines['ALLDEFINES'], which maps to a special FileAvoidWrite file generated for the PartialConfigEnvironment. MozReview-Commit-ID: 2pJ4s3TVeS8

# This file is rendered via JSON-e by
# - mozilla-taskcluster - https://docs.taskcluster.net/reference/integrations/mozilla-taskcluster/docs/taskcluster-yml
# - cron tasks - taskcluster/taskgraph/cron/decision.py
version: 1
    # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
    ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
    # ensure there's no trailing `/` on the repo URL
    repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
  - taskId: '${as_slugid("decision")}'
    taskGroupId: '${as_slugid("decision")}' # same as tsakId; this is how automation identifies a decision tsak
    schedulerId: 'gecko-level-${repository.level}'

    created: {$fromNow: ''}
    deadline: {$fromNow: '1 day'}
    expires: {$fromNow: '1 year 1 second'} # 1 second so artifacts expire first, despite rounding errors
        - owner: "${ownerEmail}"
          source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
        - $if: 'tasks_for == "hg-push"'
            name: "Gecko Decision Task"
            description: 'The task that creates all of the other tasks in the task graph'
            name: "Decision Task for cron job ${cron.job_name}"
            description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})'

    provisionerId: "aws-provisioner-v1"
    workerType: "gecko-${repository.level}-decision"

      $if: 'tasks_for == "hg-push"'
      then: {createdForUser: "${ownerEmail}"}

      $if: 'tasks_for == "hg-push"'
        - "index.gecko.v2.${repository.project}.latest.firefox.decision"
        - "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
        - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
        - "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
        - "notify.email.${ownerEmail}.on-failed"
        - "notify.email.${ownerEmail}.on-exception"
        - "index.gecko.v2.${repository.project}.latest.firefox.decision-${cron.job_name}"
        - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
        - "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"

      $if: 'tasks_for == "hg-push"'
        - 'assume:repo:${repoUrl[8:]}:*'
        - 'queue:route:notify.email.${ownerEmail}.*'
        - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'

    dependencies: []
    requires: all-completed

    priority: lowest
    retries: 5

        # checkout-gecko uses these to check out the source; the inputs
        # to `mach taskgraph decision` are all on the command line.
        GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
        GECKO_HEAD_REPOSITORY: '${repoUrl}'
        GECKO_HEAD_REF: '${push.revision}'
        GECKO_HEAD_REV: '${push.revision}'
        GECKO_COMMIT_MSG: '${push.comment}'
        HG_STORE_PATH: /builds/worker/checkouts/hg-store
        TASKCLUSTER_CACHES: /builds/worker/checkouts

        level-${repository.level}-checkouts-sparse-v1: /builds/worker/checkouts

        taskclusterProxy: true
        chainOfTrust: true

      # Note: This task is built server side without the context or tooling that
      # exist in tree so we must hard code the hash
      # XXX Changing this will break Chain of Trust without an associated puppet and
      # scriptworker patch!
      image: 'taskcluster/decision:2.0.0@sha256:4039fd878e5700b326d4a636e28c595c053fbcb53909c1db84ad1f513cf644ef'

      maxRunTime: 1800

      # TODO use mozilla-unified for the base repository once the tc-vcs
      # tar.gz archives are created or tc-vcs isn't being used.
        - /builds/worker/bin/run-task
        - '--vcs-checkout=/builds/worker/checkouts/gecko'
        - '--sparse-profile=build/sparse-profiles/taskgraph'
        - '--'
        - bash
        - -cx
        - $let:
            extraArgs: {$if: 'tasks_for == "hg-push"', then: '', else: '${cron.quoted_args}'}
          # NOTE: the explicit reference to mozilla-central below is required because android-stuff
          # still uses tc-vcs, which does not support mozilla-unified
          # https://bugzilla.mozilla.org/show_bug.cgi?id=1383973
          in: >
            cd /builds/worker/checkouts/gecko &&
            ln -s /builds/worker/artifacts artifacts &&
            ./mach --log-no-times taskgraph decision

          type: 'directory'
          path: '/builds/worker/artifacts'
          expires: {$fromNow: '1 year'}

          - machine:
              platform: gecko-decision
          - $if: 'tasks_for == "hg-push"'
              symbol: D
              groupSymbol: cron
              symbol: "${cron.job_symbol}"