author | Phil Ringnalda <philringnalda@gmail.com> |
Thu, 31 Aug 2017 20:02:34 -0700 | |
changeset 657333 | 85e0bc442b6188e1f706f7681c7a586919053294 |
parent 657332 | bd25a5e1a355a31efcdb63b3c5954746fdfb7dec |
child 657334 | 628a46720153fefa077044bff627d2f8f5b5cc17 |
push id | 77498 |
push user | hikezoe@mozilla.com |
push date | Fri, 01 Sep 2017 09:44:58 +0000 |
bugs | 1394883 |
milestone | 57.0a1 |
backs out | e3f42eca51c1c10bc3bde2884061806d0a0631c7 081f830cf28595829373dcdd04c2a90906cac043 9426705a05af11cc1698add904f8da0b6081bbd2 3a579a5054ef032fbde9691be81d7198ff686bb7 |
--- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -67,68 +67,68 @@ tasks: env: # checkout-gecko uses these to check out the source; the inputs # to `mach taskgraph decision` are all on the command line. GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified' GECKO_HEAD_REPOSITORY: '${repoUrl}' GECKO_HEAD_REF: '${push.revision}' GECKO_HEAD_REV: '${push.revision}' GECKO_COMMIT_MSG: '${push.comment}' - HG_STORE_PATH: /builds/worker/checkouts/hg-store - TASKCLUSTER_CACHES: /builds/worker/checkouts + HG_STORE_PATH: /home/worker/checkouts/hg-store + TASKCLUSTER_CACHES: /home/worker/checkouts cache: - level-${repository.level}-checkouts-sparse-v1: /builds/worker/checkouts + level-${repository.level}-checkouts-sparse-v1: /home/worker/checkouts features: taskclusterProxy: true chainOfTrust: true # Note: This task is built server side without the context or tooling that # exist in tree so we must hard code the hash # XXX Changing this will break Chain of Trust without an associated puppet and # scriptworker patch! - image: 'taskcluster/decision:2.0.0@sha256:4039fd878e5700b326d4a636e28c595c053fbcb53909c1db84ad1f513cf644ef' + image: 'taskcluster/decision:0.1.10@sha256:c5451ee6c655b3d97d4baa3b0e29a5115f23e0991d4f7f36d2a8f793076d6854' maxRunTime: 1800 # TODO use mozilla-unified for the base repository once the tc-vcs # tar.gz archives are created or tc-vcs isn't being used. command: - - /builds/worker/bin/run-task - - '--vcs-checkout=/builds/worker/checkouts/gecko' + - /home/worker/bin/run-task + - '--vcs-checkout=/home/worker/checkouts/gecko' - '--sparse-profile=build/sparse-profiles/taskgraph' - '--' - bash - -cx - $let: extraArgs: {$if: 'tasks_for == "hg-push"', then: '', else: '${cron.quoted_args}'} # NOTE: the explicit reference to mozilla-central below is required because android-stuff # still uses tc-vcs, which does not support mozilla-unified # https://bugzilla.mozilla.org/show_bug.cgi?id=1383973 in: > - cd /builds/worker/checkouts/gecko && - ln -s /builds/worker/artifacts artifacts && + cd /home/worker/checkouts/gecko && + ln -s /home/worker/artifacts artifacts && ./mach --log-no-times taskgraph decision --pushlog-id='${push.pushlog_id}' --pushdate='${push.pushdate}' --project='${repository.project}' --message="$GECKO_COMMIT_MSG" --owner='${ownerEmail}' --level='${repository.level}' --base-repository='https://hg.mozilla.org/mozilla-central' --head-repository="$GECKO_HEAD_REPOSITORY" --head-ref="$GECKO_HEAD_REF" --head-rev="$GECKO_HEAD_REV" ${extraArgs} artifacts: 'public': type: 'directory' - path: '/builds/worker/artifacts' + path: '/home/worker/artifacts' expires: {$fromNow: '1 year'} extra: treeherder: $merge: - machine: platform: gecko-decision - $if: 'tasks_for == "hg-push"'
--- a/taskcluster/ci/upload-generated-sources/kind.yml +++ b/taskcluster/ci/upload-generated-sources/kind.yml @@ -24,12 +24,12 @@ job-template: symbol: Ugs kind: build worker: docker-image: {in-tree: "lint"} max-run-time: 600 run: using: run-task command: > - cd /builds/worker/checkouts/gecko && + cd /home/worker/checkouts/gecko && ./mach python build/upload_generated_sources.py ${ARTIFACT_URL} scopes: - secrets:get:project/releng/gecko/build/level-{level}/gecko-generated-sources-upload
--- a/taskcluster/docker/README.md +++ b/taskcluster/docker/README.md @@ -90,24 +90,21 @@ Example: image: {#docker_image}builder{/docker_image} Each image has a hash and a version, given by its `HASH` and `VERSION` files. When rebuilding a prebuilt image the `VERSION` should be bumped. Once a new version of the image has been built the `HASH` file should be updated with the hash of the image. The `HASH` file is the image hash as computed by docker, this is always on the -format `sha256:<digest>`. Note that Docker produces a numbre of hashes in this -format; the hash used in this context is the one returned from `docker push`. - -In production images will be referenced by image hash. This mitigates attacks -against the registry as well as simplifying validate of correctness. The -`VERSION` file only serves to provide convenient names, such that old versions -are easy to discover in the registry (and ensuring old versions aren't deleted -by garbage-collection). +format `sha256:<digest>`. In production images will be referenced by image hash. +This mitigates attacks against the registry as well as simplifying validate of +correctness. The `VERSION` file only serves to provide convenient names, such +that old versions are easy to discover in the registry (and ensuring old +versions aren't deleted by garbage-collection). This way, older tasks which were designed to run on an older version of the image can still be executed in taskcluster, while new tasks can use the new version. Further more, this mitigates attacks against the registry as docker will verify the image hash when loading the image. Each image also has a `REGISTRY`, defaulting to the `REGISTRY` in this directory, and specifying the image registry to which the completed image should be uploaded.
--- a/taskcluster/docker/decision/Dockerfile +++ b/taskcluster/docker/decision/Dockerfile @@ -1,29 +1,28 @@ FROM ubuntu:16.04 MAINTAINER Greg Arndt <garndt@mozilla.com> # Add worker user -RUN mkdir /builds -RUN useradd -d /builds/worker -s /bin/bash -m worker -RUN mkdir /builds/worker/artifacts && chown worker:worker /builds/worker/artifacts +RUN useradd -d /home/worker -s /bin/bash -m worker +RUN mkdir /home/worker/artifacts && chown worker:worker /home/worker/artifacts # %include python/mozbuild/mozbuild/action/tooltool.py ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /tmp/tooltool.py # %include testing/mozharness/external_tools/robustcheckout.py ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py # %include taskcluster/docker/recipes/install-mercurial.sh ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /tmp/install-mercurial.sh ADD system-setup.sh /tmp/system-setup.sh RUN bash /tmp/system-setup.sh # %include taskcluster/docker/recipes/run-task -ADD topsrcdir/taskcluster/docker/recipes/run-task /builds/worker/bin/run-task +ADD topsrcdir/taskcluster/docker/recipes/run-task /home/worker/bin/run-task -ENV PATH /builds/worker/bin:$PATH +ENV PATH /home/worker/bin:$PATH ENV SHELL /bin/bash -ENV HOME /builds/worker +ENV HOME /home/worker # Set a default command useful for debugging CMD ["/bin/bash", "--login"]
--- a/taskcluster/docker/decision/HASH +++ b/taskcluster/docker/decision/HASH @@ -1,1 +1,1 @@ -sha256:4039fd878e5700b326d4a636e28c595c053fbcb53909c1db84ad1f513cf644ef +sha256:c5451ee6c655b3d97d4baa3b0e29a5115f23e0991d4f7f36d2a8f793076d6854
--- a/taskcluster/docker/decision/VERSION +++ b/taskcluster/docker/decision/VERSION @@ -1,1 +1,1 @@ -2.0.0 +0.1.10
--- a/taskcluster/docker/image_builder/Dockerfile +++ b/taskcluster/docker/image_builder/Dockerfile @@ -17,26 +17,26 @@ ADD topsrcdir/taskcluster/docker/recipes # Add and run setup script ADD build-image.sh /usr/local/bin/build-image.sh ADD download-and-compress /usr/local/bin/download-and-compress ADD setup.sh /setup/setup.sh RUN bash /setup/setup.sh # Setup a workspace that won't use AUFS. -VOLUME /builds/worker/checkouts -VOLUME /builds/worker/workspace +VOLUME /home/worker/checkouts +VOLUME /home/worker/workspace # Set variable normally configured at login, by the shells parent process, these # are taken from GNU su manual -ENV HOME /builds/worker +ENV HOME /home/worker ENV SHELL /bin/bash ENV USER worker ENV LOGNAME worker ENV HOSTNAME taskcluster-worker ENV LC_ALL C # Create worker user -RUN useradd -d /builds/worker -s /bin/bash -m worker +RUN useradd -d /home/worker -s /bin/bash -m worker # Set some sane defaults -WORKDIR /builds/worker/ +WORKDIR /home/worker/ CMD build-image.sh
--- a/taskcluster/docker/image_builder/HASH +++ b/taskcluster/docker/image_builder/HASH @@ -1,1 +1,1 @@ -sha256:24ce54a1602453bc93515aecd9d4ad25a22115fbc4b209ddb5541377e9a37315 +sha256:ceaaf92511cfbff711598005585127953873332c62f245dcf1892510c4eb371f
--- a/taskcluster/docker/image_builder/VERSION +++ b/taskcluster/docker/image_builder/VERSION @@ -1,1 +1,1 @@ -2.0.0 +1.3.0
--- a/taskcluster/docker/image_builder/build-image.sh +++ b/taskcluster/docker/image_builder/build-image.sh @@ -13,31 +13,31 @@ raise_error() { exit 1 } # Ensure that the PROJECT is specified so the image can be indexed test -n "$PROJECT" || raise_error "PROJECT must be provided." test -n "$HASH" || raise_error "Context HASH must be provided." test -n "$IMAGE_NAME" || raise_error "IMAGE_NAME must be provided." +# Create artifact folder +mkdir -p /home/worker/workspace/artifacts + # Construct a CONTEXT_FILE -CONTEXT_FILE=/builds/worker/workspace/context.tar +CONTEXT_FILE=/home/worker/workspace/context.tar # Run ./mach taskcluster-build-image with --context-only to build context run-task \ - --vcs-checkout "/builds/worker/checkouts/gecko" \ + --vcs-checkout "/home/worker/checkouts/gecko" \ -- \ - /builds/worker/checkouts/gecko/mach taskcluster-build-image \ + /home/worker/checkouts/gecko/mach taskcluster-build-image \ --context-only "$CONTEXT_FILE" \ "$IMAGE_NAME" test -f "$CONTEXT_FILE" || raise_error "Context file wasn't created" -# Create artifact folder (note that this must occur after run-task) -mkdir -p /builds/worker/workspace/artifacts - # Post context tar-ball to docker daemon # This interacts directly with the docker remote API, see: # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.18/ curl -s --fail \ -X POST \ --header 'Content-Type: application/tar' \ --data-binary "@$CONTEXT_FILE" \ --unix-socket /var/run/docker.sock "http:/build?t=$IMAGE_NAME:$HASH" \ @@ -58,10 +58,10 @@ fi # Get image from docker daemon (try up to 10 times) # This interacts directly with the docker remote API, see: # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.18/ # # The script will retry up to 10 times. /usr/local/bin/download-and-compress \ http+unix://%2Fvar%2Frun%2Fdocker.sock/images/${IMAGE_NAME}:${HASH}/get \ - /builds/worker/workspace/image.tar.zst.tmp \ - /builds/worker/workspace/artifacts/image.tar.zst + /home/worker/workspace/image.tar.zst.tmp \ + /home/worker/workspace/artifacts/image.tar.zst
--- a/taskcluster/docker/image_builder/setup.sh +++ b/taskcluster/docker/image_builder/setup.sh @@ -24,17 +24,17 @@ apt-get install -y python-pip . /setup/install-mercurial.sh # Install build-image.sh script chmod +x /usr/local/bin/build-image.sh chmod +x /usr/local/bin/run-task chmod +x /usr/local/bin/download-and-compress # Create workspace -mkdir -p /builds/worker/workspace +mkdir -p /home/worker/workspace # Install python-zstandard. cd /setup tooltool_fetch <<EOF [ { "size": 463794, "visibility": "public",
--- a/taskcluster/docs/docker-images.rst +++ b/taskcluster/docs/docker-images.rst @@ -3,18 +3,16 @@ ============= Docker Images ============= TaskCluster Docker images are defined in the source directory under ``taskcluster/docker``. Each directory therein contains the name of an image used as part of the task graph. -More information is available in the ``README.md`` file in that directory. - Adding Extra Files to Images ============================ Dockerfile syntax has been extended to allow *any* file from the source checkout to be added to the image build *context*. (Traditionally you can only ``ADD`` files from the same directory as the Dockerfile.) Simply add the following syntax as a comment in a Dockerfile:: @@ -36,9 +34,9 @@ context under the ``topsrcdir/`` path. Files are added as they exist on disk. e.g. executable flags should be preserved. However, the file owner/group is changed to ``root`` and the ``mtime`` of the file is normalized. Here is an example Dockerfile snippet:: # %include mach - ADD topsrcdir/mach /builds/worker/mach + ADD topsrcdir/mach /home/worker/mach
--- a/taskcluster/taskgraph/action.yml +++ b/taskcluster/taskgraph/action.yml @@ -33,17 +33,17 @@ payload: cache: level-{{level}}-checkouts: /home/worker/checkouts features: taskclusterProxy: true # Note: This task is built server side without the context or tooling that # exist in tree so we must hard code the version - image: 'taskcluster/decision:2.0.0@sha256:4039fd878e5700b326d4a636e28c595c053fbcb53909c1db84ad1f513cf644ef' + image: 'taskcluster/decision:0.1.7' # Virtually no network or other potentially risky operations happen as part # of the task timeout aside from the initial clone. We intentionally have # set this to a lower value _all_ decision tasks should use a root # repository which is cached. maxRunTime: 1800 command:
--- a/taskcluster/taskgraph/transforms/docker_image.py +++ b/taskcluster/taskgraph/transforms/docker_image.py @@ -95,38 +95,37 @@ def fill_template(config, tasks): # this image.. 'worker': { 'implementation': 'docker-worker', 'os': 'linux', 'docker-image': docker_image('image_builder'), 'caches': [{ 'type': 'persistent', 'name': 'level-{}-imagebuilder-v1'.format(config.params['level']), - 'mount-point': '/builds/worker/checkouts', + 'mount-point': '/home/worker/checkouts', }], 'volumes': [ - # Keep in sync with Dockerfile and TASKCLUSTER_VOLUMES - '/builds/worker/checkouts', - '/builds/worker/workspace', + # Keep in sync with Dockerfile. + '/home/worker/checkouts', + '/home/worker/workspace', ], 'artifacts': [{ 'type': 'file', - 'path': '/builds/worker/workspace/artifacts/image.tar.zst', + 'path': '/home/worker/workspace/artifacts/image.tar.zst', 'name': 'public/image.tar.zst', }], 'env': { - 'HG_STORE_PATH': '/builds/worker/checkouts/hg-store', + 'HG_STORE_PATH': '/home/worker/checkouts/hg-store', 'HASH': context_hash, 'PROJECT': config.params['project'], 'IMAGE_NAME': image_name, 'DOCKER_IMAGE_ZSTD_LEVEL': zstd_level, 'GECKO_BASE_REPOSITORY': config.params['base_repository'], 'GECKO_HEAD_REPOSITORY': config.params['head_repository'], 'GECKO_HEAD_REV': config.params['head_rev'], - 'TASKCLUSTER_VOLUMES': '/builds/worker/checkouts;/builds/worker/workspace', }, 'chain-of-trust': True, 'docker-in-docker': True, 'taskcluster-proxy': True, 'max-run-time': 7200, }, }