author | Sebastian Hengst <archaeopteryx@coole-files.de> |
Mon, 19 Dec 2016 21:15:07 +0100 | |
changeset 371435 | 9d0c5328c89782f32bb458487278944bf219a417 |
parent 371434 | 3b0dd252edd7efc2da801ff8f141473fbc90193a |
child 371436 | b52c0a02c07b67b8049ee7259fb182edec5dc8cc |
push id | 6996 |
push user | jlorenzo@mozilla.com |
push date | Mon, 06 Mar 2017 20:48:21 +0000 |
treeherder | mozilla-beta@d89512dab048 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | backout |
bugs | 1302763 |
milestone | 53.0a1 |
backs out | fe7303de56be76914f0eebe008b0ef02c803b6cd |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/AUTHORS +++ b/AUTHORS @@ -874,17 +874,16 @@ Roy Yokoyama <yokoyama@netscape.com> RSA Security, Inc Russell King <rmk@arm.linux.org.uk> Rusty Lynch <rusty.lynch@intel.com> Ryan Cassin <rcassin@supernova.org> Ryan Flint <rflint@dslr.net> Ryan Jones <sciguyryan@gmail.com> Ryan VanderMeulen <ryanvm@gmail.com> Ryoichi Furukawa <oliver@1000cp.com> -Sanyam Khurana <Sanyam.Khurana01@gmail.com> sagdjb@softwareag.com Samir Gehani <sgehani@netscape.com> Sammy Ford Samphan Raruenrom Samuel Sieb <samuel@sieb.net> Sarlos Tamas scole@planetweb.com Scooter Morris <scootermorris@comcast.net>
--- a/taskcluster/ci/android-stuff/kind.yml +++ b/taskcluster/ci/android-stuff/kind.yml @@ -58,17 +58,17 @@ jobs: - "/home/worker/bin/before.sh && /home/worker/bin/build.sh && /home/worker/bin/after.sh && true\n" max-run-time: 36000 scopes: - docker-worker:relengapi-proxy:tooltool.download.internal - docker-worker:relengapi-proxy:tooltool.download.public when: files-changed: - "mobile/android/config/**" - - "taskcluster/docker/android-gradle-build/**" + - "testing/docker/android-gradle-build/**" - "testing/mozharness/configs/builds/releng_sub_android_configs/*gradle_dependencies.py" - "**/*.gradle" android-test: description: "Android armv7 unit tests" attributes: build_platform: android-test build_type: opt
--- a/taskcluster/ci/docker-image/kind.yml +++ b/taskcluster/ci/docker-image/kind.yml @@ -1,14 +1,14 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. implementation: 'taskgraph.task.docker_image:DockerImageTask' -images_path: '../../../taskcluster/docker' +images_path: '../../../testing/docker' # make a task for each docker-image we might want. For the moment, since we # write artifacts for each, these are whitelisted, but ideally that will change # (to use subdirectory clones of the proper directory), at which point we can # generate tasks for every docker image in the directory, secure in the # knowledge that unnecessary images will be omitted from the target task graph images: desktop-test: dt
--- a/taskcluster/ci/source-check/mozlint.yml +++ b/taskcluster/ci/source-check/mozlint.yml @@ -35,17 +35,17 @@ mozlint-eslint/opt: # Run when eslint policies change. - '**/.eslintignore' - '**/*eslintrc*' # The plugin implementing custom checks. - 'tools/lint/eslint/eslint-plugin-mozilla/**' # Other misc lint related files. - 'python/mozlint/**' - 'tools/lint/**' - - 'taskcluster/docker/lint/**' + - 'testing/docker/lint/**' mozlint-flake8/opt: description: flake8 run over the gecko codebase treeherder: symbol: f8 kind: test tier: 1 platform: lint/opt @@ -61,17 +61,17 @@ mozlint-flake8/opt: - integration - release when: files-changed: - '**/*.py' - '**/.flake8' - 'python/mozlint/**' - 'tools/lint/**' - - 'taskcluster/docker/lint/**' + - 'testing/docker/lint/**' wptlint-gecko/opt: description: web-platform-tests linter treeherder: symbol: W kind: test tier: 1 platform: lint/opt @@ -89,9 +89,9 @@ wptlint-gecko/opt: when: files-changed: - 'testing/web-platform/tests/**' - 'testing/web-platform/mozilla/tests/**' - 'testing/web-platform/meta/MANIFEST.json' - 'testing/web-platform/mozilla/meta/MANIFEST.json' - 'python/mozlint/**' - 'tools/lint/**' - - 'taskcluster/docker/lint/**' + - 'testing/docker/lint/**'
--- a/taskcluster/docs/docker-images.rst +++ b/taskcluster/docs/docker-images.rst @@ -1,16 +1,16 @@ .. taskcluster_dockerimages: ============= Docker Images ============= TaskCluster Docker images are defined in the source directory under -``taskcluster/docker``. Each directory therein contains the name of an +``testing/docker``. Each directory therein contains the name of an image used as part of the task graph. Adding Extra Files to Images ============================ Dockerfile syntax has been extended to allow *any* file from the source checkout to be added to the image build *context*. (Traditionally you can only ``ADD`` files from the same directory as the Dockerfile.)
--- a/taskcluster/docs/kinds.rst +++ b/taskcluster/docs/kinds.rst @@ -133,17 +133,17 @@ docker-image ------------ Tasks of the ``docker-image`` kind build the Docker images in which other Docker tasks run. The tasks to generate each docker image have predictable labels: ``build-docker-image-<name>``. -Docker images are built from subdirectories of ``taskcluster/docker``, using +Docker images are built from subdirectories of ``testing/docker``, using ``docker build``. There is currently no capability for one Docker image to depend on another in-tree docker image, without uploading the latter to a Docker repository The task definition used to create the image-building tasks is given in ``image.yml`` in the kind directory, and is interpreted as a :doc:`YAML Template <yaml-templates>`.
--- a/taskcluster/scripts/builder/build-haz-linux.sh +++ b/taskcluster/scripts/builder/build-haz-linux.sh @@ -49,17 +49,17 @@ GECKO_DIR=$( cd "$GECKO_DIR" && pwd ) # Directory to populate with tooltool-installed tools export TOOLTOOL_DIR="$WORKSPACE" # Directory to hold the (useless) object files generated by the analysis. export MOZ_OBJDIR="$WORKSPACE/obj-analyzed" mkdir -p "$MOZ_OBJDIR" if [ -n "$DO_TOOLTOOL" ]; then - ( cd $TOOLTOOL_DIR; python $GECKO_DIR/taskcluster/docker/recipes/tooltool.py --url https://api.pub.build.mozilla.org/tooltool/ -m $GECKO_DIR/$TOOLTOOL_MANIFEST fetch -c $TOOLTOOL_CACHE ) + ( cd $TOOLTOOL_DIR; python $GECKO_DIR/testing/docker/recipes/tooltool.py --url https://api.pub.build.mozilla.org/tooltool/ -m $GECKO_DIR/$TOOLTOOL_MANIFEST fetch -c $TOOLTOOL_CACHE ) fi export NO_MERCURIAL_SETUP_CHECK=1 if [[ "$PROJECT" = "browser" ]]; then ( cd "$WORKSPACE" set "$WORKSPACE" . setup-ccache.sh
--- a/taskcluster/scripts/builder/sm-tooltool-config.sh +++ b/taskcluster/scripts/builder/sm-tooltool-config.sh @@ -42,9 +42,9 @@ esac # necessary for the JS shell, but it's less duplication to share tooltool # manifests. BROWSER_PLATFORM=$PLATFORM_OS$BITS : ${TOOLTOOL_MANIFEST:=browser/config/tooltool-manifests/$BROWSER_PLATFORM/releng.manifest} : ${TOOLTOOL_CHECKOUT:=$WORK} export TOOLTOOL_CHECKOUT -(cd $TOOLTOOL_CHECKOUT && python ${SRCDIR}/taskcluster/docker/recipes/tooltool.py --url $TOOLTOOL_SERVER -m $SRCDIR/$TOOLTOOL_MANIFEST fetch ${TOOLTOOL_CACHE:+ -c $TOOLTOOL_CACHE}) +(cd $TOOLTOOL_CHECKOUT && python ${SRCDIR}/testing/docker/recipes/tooltool.py --url $TOOLTOOL_SERVER -m $SRCDIR/$TOOLTOOL_MANIFEST fetch ${TOOLTOOL_CACHE:+ -c $TOOLTOOL_CACHE})
--- a/taskcluster/taskgraph/docker.py +++ b/taskcluster/taskgraph/docker.py @@ -15,23 +15,23 @@ import tempfile import urllib2 import which from subprocess import Popen, PIPE from io import BytesIO from taskgraph.util import docker GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..')) -IMAGE_DIR = os.path.join(GECKO, 'taskcluster', 'docker') +IMAGE_DIR = os.path.join(GECKO, 'testing', 'docker') INDEX_URL = 'https://index.taskcluster.net/v1/task/' + docker.INDEX_PREFIX + '.{}.{}.hash.{}' ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}' def load_image_by_name(image_name, tag=None): - context_path = os.path.join(GECKO, 'taskcluster', 'docker', image_name) + context_path = os.path.join(GECKO, 'testing', 'docker', image_name) context_hash = docker.generate_context_hash(GECKO, context_path, image_name) image_index_url = INDEX_URL.format('level-3', image_name, context_hash) print("Fetching", image_index_url) task = json.load(urllib2.urlopen(image_index_url)) return load_image_by_task_id(task['taskId'], tag)
--- a/taskcluster/taskgraph/task/docker_image.py +++ b/taskcluster/taskgraph/task/docker_image.py @@ -54,17 +54,17 @@ class DockerImageTask(base.Task): .format(repo=params['head_repository'], rev=params['head_rev']), 'index_image_prefix': INDEX_PREFIX, 'artifact_path': 'public/image.tar.zst', } tasks = [] templates = Templates(path) for image_name, image_symbol in config['images'].iteritems(): - context_path = os.path.join('taskcluster', 'docker', image_name) + context_path = os.path.join('testing', 'docker', image_name) context_hash = generate_context_hash(GECKO, context_path, image_name) image_parameters = dict(parameters) image_parameters['image_name'] = image_name image_parameters['context_hash'] = context_hash image_task = templates.load('image.yml', image_parameters) attributes = {'image_name': image_name}
--- a/taskcluster/taskgraph/transforms/task.py +++ b/taskcluster/taskgraph/transforms/task.py @@ -136,17 +136,17 @@ task_description_schema = Schema({ # For tasks that will run in docker-worker or docker-engine, this is the # name of the docker image or in-tree docker image to run the task in. If # in-tree, then a dependency will be created automatically. This is # generally `desktop-test`, or an image that acts an awful lot like it. Required('docker-image'): Any( # a raw Docker image path (repo/image:tag) basestring, - # an in-tree generated docker image (from `taskcluster/docker/<name>`) + # an in-tree generated docker image (from `testing/docker/<name>`) {'in-tree': basestring} ), # worker features that should be enabled Required('relengapi-proxy', default=False): bool, Required('chain-of-trust', default=False): bool, Required('taskcluster-proxy', default=False): bool, Required('allow-ptrace', default=False): bool,
--- a/taskcluster/taskgraph/transforms/tests/test_description.py +++ b/taskcluster/taskgraph/transforms/tests/test_description.py @@ -121,17 +121,17 @@ test_description_schema = Schema({ # For tasks that will run in docker-worker or docker-engine, this is the # name of the docker image or in-tree docker image to run the task in. If # in-tree, then a dependency will be created automatically. This is # generally `desktop-test`, or an image that acts an awful lot like it. Required('docker-image', default={'in-tree': 'desktop-test'}): Any( # a raw Docker image path (repo/image:tag) basestring, - # an in-tree generated docker image (from `taskcluster/docker/<name>`) + # an in-tree generated docker image (from `testing/docker/<name>`) {'in-tree': basestring} ), # seconds of runtime after which the task will be killed. Like 'chunks', # this can be keyed by test pltaform. Required('max-run-time', default=3600): Any( int, {'by-test-platform': {basestring: int}},
--- a/taskcluster/taskgraph/util/docker.py +++ b/taskcluster/taskgraph/util/docker.py @@ -12,17 +12,17 @@ import tarfile import tempfile from mozpack.archive import ( create_tar_gz_from_files, ) GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..')) -DOCKER_ROOT = os.path.join(GECKO, 'taskcluster', 'docker') +DOCKER_ROOT = os.path.join(GECKO, 'testing', 'docker') INDEX_PREFIX = 'docker.images.v2' ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}' def docker_image(name, default_version=None): '''Determine the docker image name, including repository and tag, from an in-tree docker file.''' try:
rename from taskcluster/docker/android-gradle-build/Dockerfile rename to testing/docker/android-gradle-build/Dockerfile --- a/taskcluster/docker/android-gradle-build/Dockerfile +++ b/testing/docker/android-gradle-build/Dockerfile @@ -9,18 +9,18 @@ VOLUME /home/worker/workspace VOLUME /home/worker/tooltool-cache # Add build scripts; these are the entry points from the taskcluster worker, and # operate on environment variables ADD bin /home/worker/bin RUN chmod +x /home/worker/bin/* # Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb -# %include taskcluster/docker/recipes/xvfb.sh -ADD topsrcdir/taskcluster/docker/recipes/xvfb.sh /home/worker/scripts/xvfb.sh +# %include testing/docker/recipes/xvfb.sh +ADD topsrcdir/testing/docker/recipes/xvfb.sh /home/worker/scripts/xvfb.sh # Add configuration COPY dot-config /home/worker/.config # Generate machine uuid file RUN dbus-uuidgen --ensure=/var/lib/dbus/machine-id # Stubbed out credentials; mozharness looks for this file an issues a WARNING
rename from taskcluster/docker/android-gradle-build/README.md rename to testing/docker/android-gradle-build/README.md
rename from taskcluster/docker/android-gradle-build/REGISTRY rename to testing/docker/android-gradle-build/REGISTRY
rename from taskcluster/docker/android-gradle-build/VERSION rename to testing/docker/android-gradle-build/VERSION
rename from taskcluster/docker/android-gradle-build/bin/after.sh rename to testing/docker/android-gradle-build/bin/after.sh
rename from taskcluster/docker/android-gradle-build/bin/before.sh rename to testing/docker/android-gradle-build/bin/before.sh
rename from taskcluster/docker/android-gradle-build/bin/build.sh rename to testing/docker/android-gradle-build/bin/build.sh
rename from taskcluster/docker/android-gradle-build/bin/checkout-script.sh rename to testing/docker/android-gradle-build/bin/checkout-script.sh
rename from taskcluster/docker/android-gradle-build/bin/checkout-sources.sh rename to testing/docker/android-gradle-build/bin/checkout-sources.sh
rename from taskcluster/docker/android-gradle-build/buildprops.json rename to testing/docker/android-gradle-build/buildprops.json
rename from taskcluster/docker/android-gradle-build/dot-config/pip/pip.conf rename to testing/docker/android-gradle-build/dot-config/pip/pip.conf
rename from taskcluster/docker/android-gradle-build/oauth.txt rename to testing/docker/android-gradle-build/oauth.txt
rename from taskcluster/docker/base-build/system-setup.sh rename to testing/docker/base-build/system-setup.sh
rename from taskcluster/docker/base-test/sources.list rename to testing/docker/base-test/sources.list
rename from taskcluster/docker/centos6-build-upd/Dockerfile rename to testing/docker/centos6-build-upd/Dockerfile
rename from taskcluster/docker/centos6-build-upd/REGISTRY rename to testing/docker/centos6-build-upd/REGISTRY
rename from taskcluster/docker/centos6-build-upd/VERSION rename to testing/docker/centos6-build-upd/VERSION
rename from taskcluster/docker/centos6-build/Dockerfile rename to testing/docker/centos6-build/Dockerfile
rename from taskcluster/docker/centos6-build/REGISTRY rename to testing/docker/centos6-build/REGISTRY
rename from taskcluster/docker/centos6-build/system-setup.sh rename to testing/docker/centos6-build/system-setup.sh
rename from taskcluster/docker/decision/Dockerfile rename to testing/docker/decision/Dockerfile --- a/taskcluster/docker/decision/Dockerfile +++ b/testing/docker/decision/Dockerfile @@ -1,28 +1,28 @@ FROM ubuntu:16.04 MAINTAINER Greg Arndt <garndt@mozilla.com> # Add worker user RUN useradd -d /home/worker -s /bin/bash -m worker RUN mkdir /home/worker/artifacts && chown worker:worker /home/worker/artifacts -# %include taskcluster/docker/recipes/tooltool.py -ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /tmp/tooltool.py +# %include testing/docker/recipes/tooltool.py +ADD topsrcdir/testing/docker/recipes/tooltool.py /tmp/tooltool.py # %include testing/mozharness/external_tools/robustcheckout.py ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py -# %include taskcluster/docker/recipes/install-mercurial.sh -ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /tmp/install-mercurial.sh +# %include testing/docker/recipes/install-mercurial.sh +ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /tmp/install-mercurial.sh ADD system-setup.sh /tmp/system-setup.sh RUN bash /tmp/system-setup.sh -# %include taskcluster/docker/recipes/run-task -ADD topsrcdir/taskcluster/docker/recipes/run-task /home/worker/bin/run-task +# %include testing/docker/recipes/run-task +ADD topsrcdir/testing/docker/recipes/run-task /home/worker/bin/run-task ENV PATH /home/worker/bin:$PATH ENV SHELL /bin/bash ENV HOME /home/worker # Set a default command useful for debugging CMD ["/bin/bash", "--login"]
rename from taskcluster/docker/decision/system-setup.sh rename to testing/docker/decision/system-setup.sh
rename from taskcluster/docker/desktop-build/Dockerfile rename to testing/docker/desktop-build/Dockerfile --- a/taskcluster/docker/desktop-build/Dockerfile +++ b/testing/docker/desktop-build/Dockerfile @@ -6,43 +6,43 @@ MAINTAINER Dustin J. Mitchell <dustin VOLUME /home/worker/workspace VOLUME /home/worker/tooltool-cache # Add build scripts; these are the entry points from the taskcluster worker, and # operate on environment variables ADD bin /home/worker/bin RUN chmod +x /home/worker/bin/* -# %include taskcluster/docker/recipes/tooltool.py -ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /builds/tooltool.py -ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /setup/tooltool.py +# %include testing/docker/recipes/tooltool.py +ADD topsrcdir/testing/docker/recipes/tooltool.py /builds/tooltool.py +ADD topsrcdir/testing/docker/recipes/tooltool.py /setup/tooltool.py # %include testing/mozharness/external_tools/robustcheckout.py ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py -# %include taskcluster/docker/recipes/common.sh -ADD topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh +# %include testing/docker/recipes/common.sh +ADD topsrcdir/testing/docker/recipes/common.sh /setup/common.sh -# %include taskcluster/docker/recipes/install-mercurial.sh -ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh +# %include testing/docker/recipes/install-mercurial.sh +ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh -# %include taskcluster/docker/recipes/centos6-build-system-setup.sh -ADD topsrcdir/taskcluster/docker/recipes/centos6-build-system-setup.sh /setup/system-setup.sh +# %include testing/docker/recipes/centos6-build-system-setup.sh +ADD topsrcdir/testing/docker/recipes/centos6-build-system-setup.sh /setup/system-setup.sh # TODO remove once base image doesn't install Mercurial RUN pip uninstall -y Mercurial RUN bash /setup/system-setup.sh # Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb -# %include taskcluster/docker/recipes/xvfb.sh -ADD topsrcdir/taskcluster/docker/recipes/xvfb.sh /home/worker/scripts/xvfb.sh +# %include testing/docker/recipes/xvfb.sh +ADD topsrcdir/testing/docker/recipes/xvfb.sh /home/worker/scripts/xvfb.sh -# %include taskcluster/docker/recipes/run-task -ADD topsrcdir/taskcluster/docker/recipes/run-task /home/worker/bin/run-task +# %include testing/docker/recipes/run-task +ADD topsrcdir/testing/docker/recipes/run-task /home/worker/bin/run-task # Add configuration COPY dot-config /home/worker/.config # Generate machine uuid file RUN dbus-uuidgen --ensure=/var/lib/dbus/machine-id # Stubbed out credentials; mozharness looks for this file an issues a WARNING
rename from taskcluster/docker/desktop-build/bin/build.sh rename to testing/docker/desktop-build/bin/build.sh
rename from taskcluster/docker/desktop-build/bin/checkout-script.sh rename to testing/docker/desktop-build/bin/checkout-script.sh
rename from taskcluster/docker/desktop-build/bin/checkout-sources.sh rename to testing/docker/desktop-build/bin/checkout-sources.sh
rename from taskcluster/docker/desktop-build/buildprops.json rename to testing/docker/desktop-build/buildprops.json
rename from taskcluster/docker/desktop-build/dot-config/pip/pip.conf rename to testing/docker/desktop-build/dot-config/pip/pip.conf
rename from taskcluster/docker/desktop-build/oauth.txt rename to testing/docker/desktop-build/oauth.txt
rename from taskcluster/docker/desktop-test/Dockerfile rename to testing/docker/desktop-test/Dockerfile --- a/taskcluster/docker/desktop-test/Dockerfile +++ b/testing/docker/desktop-test/Dockerfile @@ -1,37 +1,37 @@ FROM ubuntu:12.04 MAINTAINER Jonas Finnemann Jensen <jopsen@gmail.com> RUN useradd -d /home/worker -s /bin/bash -m worker WORKDIR /home/worker -# %include taskcluster/docker/recipes/tooltool.py -ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /setup/tooltool.py +# %include testing/docker/recipes/tooltool.py +ADD topsrcdir/testing/docker/recipes/tooltool.py /setup/tooltool.py # %include testing/mozharness/external_tools/robustcheckout.py ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py -# %include taskcluster/docker/recipes/install-mercurial.sh -ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /tmp/install-mercurial.sh +# %include testing/docker/recipes/install-mercurial.sh +ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /tmp/install-mercurial.sh # Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb -# %include taskcluster/docker/recipes/xvfb.sh -ADD topsrcdir/taskcluster/docker/recipes/xvfb.sh /home/worker/scripts/xvfb.sh +# %include testing/docker/recipes/xvfb.sh +ADD topsrcdir/testing/docker/recipes/xvfb.sh /home/worker/scripts/xvfb.sh # Add the tooltool manifest containing the minidump_stackwalk binary. # %include testing/config/tooltool-manifests/linux64/releng.manifest ADD topsrcdir/testing/config/tooltool-manifests/linux64/releng.manifest /tmp/minidump_stackwalk.manifest -# %include taskcluster/docker/recipes/ubuntu1204-test-system-setup.sh -ADD topsrcdir/taskcluster/docker/recipes/ubuntu1204-test-system-setup.sh /setup/system-setup.sh +# %include testing/docker/recipes/ubuntu1204-test-system-setup.sh +ADD topsrcdir/testing/docker/recipes/ubuntu1204-test-system-setup.sh /setup/system-setup.sh RUN bash /setup/system-setup.sh -# %include taskcluster/docker/recipes/run-task -ADD topsrcdir/taskcluster/docker/recipes/run-task /home/worker/bin/run-task +# %include testing/docker/recipes/run-task +ADD topsrcdir/testing/docker/recipes/run-task /home/worker/bin/run-task # %include taskcluster/scripts/tester/test-ubuntu.sh ADD topsrcdir/taskcluster/scripts/tester/test-ubuntu.sh /home/worker/bin/test-linux.sh # This will create a host mounted filesystem when the cache is stripped # on Try. This cancels out some of the performance losses of aufs. See # bug 1291940. VOLUME /home/worker/checkouts
rename from taskcluster/docker/desktop-test/buildprops.json rename to testing/docker/desktop-test/buildprops.json
rename from taskcluster/docker/desktop-test/deja-dup-monitor.desktop rename to testing/docker/desktop-test/deja-dup-monitor.desktop
rename from taskcluster/docker/desktop-test/dot-files/config/pip/pip.conf rename to testing/docker/desktop-test/dot-files/config/pip/pip.conf
rename from taskcluster/docker/desktop-test/dot-files/config/user-dirs.dirs rename to testing/docker/desktop-test/dot-files/config/user-dirs.dirs
rename from taskcluster/docker/desktop-test/dot-files/config/user-dirs.locale rename to testing/docker/desktop-test/dot-files/config/user-dirs.locale
rename from taskcluster/docker/desktop-test/dot-files/pulse/default.pa rename to testing/docker/desktop-test/dot-files/pulse/default.pa
rename from taskcluster/docker/desktop-test/fonts.conf rename to testing/docker/desktop-test/fonts.conf
rename from taskcluster/docker/desktop-test/jockey-gtk.desktop rename to testing/docker/desktop-test/jockey-gtk.desktop
rename from taskcluster/docker/desktop-test/release-upgrades rename to testing/docker/desktop-test/release-upgrades
rename from taskcluster/docker/desktop-test/taskcluster-interactive-shell rename to testing/docker/desktop-test/taskcluster-interactive-shell
rename from taskcluster/docker/desktop-test/tc-vcs-config.yml rename to testing/docker/desktop-test/tc-vcs-config.yml
rename from taskcluster/docker/desktop-test/tester.env rename to testing/docker/desktop-test/tester.env
rename from taskcluster/docker/desktop1604-test/Dockerfile rename to testing/docker/desktop1604-test/Dockerfile --- a/taskcluster/docker/desktop1604-test/Dockerfile +++ b/testing/docker/desktop1604-test/Dockerfile @@ -1,40 +1,40 @@ FROM ubuntu:16.04 MAINTAINER Joel Maher <joel.maher@gmail.com> RUN useradd -d /home/worker -s /bin/bash -m worker WORKDIR /home/worker -# %include taskcluster/docker/recipes/tooltool.py -ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /setup/tooltool.py +# %include testing/docker/recipes/tooltool.py +ADD topsrcdir/testing/docker/recipes/tooltool.py /setup/tooltool.py # %include testing/mozharness/external_tools/robustcheckout.py ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py -# %include taskcluster/docker/recipes/common.sh -ADD topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh +# %include testing/docker/recipes/common.sh +ADD topsrcdir/testing/docker/recipes/common.sh /setup/common.sh -# %include taskcluster/docker/recipes/install-mercurial.sh -ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh +# %include testing/docker/recipes/install-mercurial.sh +ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh # Add the tooltool manifest containing the minidump_stackwalk binary. # %include testing/config/tooltool-manifests/linux64/releng.manifest ADD topsrcdir/testing/config/tooltool-manifests/linux64/releng.manifest /tmp/minidump_stackwalk.manifest -# %include taskcluster/docker/recipes/ubuntu1604-test-system-setup.sh -ADD topsrcdir/taskcluster/docker/recipes/ubuntu1604-test-system-setup.sh /setup/system-setup.sh +# %include testing/docker/recipes/ubuntu1604-test-system-setup.sh +ADD topsrcdir/testing/docker/recipes/ubuntu1604-test-system-setup.sh /setup/system-setup.sh RUN bash /setup/system-setup.sh # Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb -# %include taskcluster/docker/recipes/xvfb.sh -ADD topsrcdir/taskcluster/docker/recipes/xvfb.sh /home/worker/scripts/xvfb.sh +# %include testing/docker/recipes/xvfb.sh +ADD topsrcdir/testing/docker/recipes/xvfb.sh /home/worker/scripts/xvfb.sh -# %include taskcluster/docker/recipes/run-task -ADD topsrcdir/taskcluster/docker/recipes/run-task /home/worker/bin/run-task +# %include testing/docker/recipes/run-task +ADD topsrcdir/testing/docker/recipes/run-task /home/worker/bin/run-task # %include taskcluster/scripts/tester/test-ubuntu.sh ADD topsrcdir/taskcluster/scripts/tester/test-ubuntu.sh /home/worker/bin/test-linux.sh # This will create a host mounted filesystem when the cache is stripped # on Try. This cancels out some of the performance losses of aufs. See # bug 1291940. VOLUME /home/worker/checkouts
rename from taskcluster/docker/desktop1604-test/apport rename to testing/docker/desktop1604-test/apport
rename from taskcluster/docker/desktop1604-test/buildprops.json rename to testing/docker/desktop1604-test/buildprops.json
rename from taskcluster/docker/desktop1604-test/deja-dup-monitor.desktop rename to testing/docker/desktop1604-test/deja-dup-monitor.desktop
rename from taskcluster/docker/desktop1604-test/dot-files/config/pip/pip.conf rename to testing/docker/desktop1604-test/dot-files/config/pip/pip.conf
rename from taskcluster/docker/desktop1604-test/dot-files/config/user-dirs.dirs rename to testing/docker/desktop1604-test/dot-files/config/user-dirs.dirs
rename from taskcluster/docker/desktop1604-test/dot-files/config/user-dirs.locale rename to testing/docker/desktop1604-test/dot-files/config/user-dirs.locale
rename from taskcluster/docker/desktop1604-test/dot-files/pulse/default.pa rename to testing/docker/desktop1604-test/dot-files/pulse/default.pa
rename from taskcluster/docker/desktop1604-test/fonts.conf rename to testing/docker/desktop1604-test/fonts.conf
rename from taskcluster/docker/desktop1604-test/jockey-gtk.desktop rename to testing/docker/desktop1604-test/jockey-gtk.desktop
rename from taskcluster/docker/desktop1604-test/release-upgrades rename to testing/docker/desktop1604-test/release-upgrades
rename from taskcluster/docker/desktop1604-test/taskcluster-interactive-shell rename to testing/docker/desktop1604-test/taskcluster-interactive-shell
rename from taskcluster/docker/desktop1604-test/tc-vcs-config.yml rename to testing/docker/desktop1604-test/tc-vcs-config.yml
rename from taskcluster/docker/desktop1604-test/tester.env rename to testing/docker/desktop1604-test/tester.env
rename from taskcluster/docker/image_builder/Dockerfile rename to testing/docker/image_builder/Dockerfile --- a/taskcluster/docker/image_builder/Dockerfile +++ b/testing/docker/image_builder/Dockerfile @@ -1,24 +1,24 @@ FROM ubuntu:16.04 -# %include taskcluster/docker/recipes/tooltool.py -ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /setup/tooltool.py +# %include testing/docker/recipes/tooltool.py +ADD topsrcdir/testing/docker/recipes/tooltool.py /setup/tooltool.py -# %include taskcluster/docker/recipes/common.sh -ADD topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh +# %include testing/docker/recipes/common.sh +ADD topsrcdir/testing/docker/recipes/common.sh /setup/common.sh -# %include taskcluster/docker/recipes/install-mercurial.sh -ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh +# %include testing/docker/recipes/install-mercurial.sh +ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh -# %include taskcluster/mozharness/external_tools/robustcheckout.py -ADD topsrcdir/taskcluster/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py +# %include testing/mozharness/external_tools/robustcheckout.py +ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py -# %include taskcluster/docker/recipes/run-task -ADD topsrcdir/taskcluster/docker/recipes/run-task /usr/local/bin/run-task +# %include testing/docker/recipes/run-task +ADD topsrcdir/testing/docker/recipes/run-task /usr/local/bin/run-task # Add and run setup script ADD build-image.sh /usr/local/bin/build-image.sh ADD setup.sh /setup/setup.sh RUN bash /setup/setup.sh # Setup a workspace that won't use AUFS VOLUME /home/worker/workspace
rename from taskcluster/docker/image_builder/REGISTRY rename to testing/docker/image_builder/REGISTRY
rename from taskcluster/docker/image_builder/build-image.sh rename to testing/docker/image_builder/build-image.sh
rename from taskcluster/docker/image_builder/setup.sh rename to testing/docker/image_builder/setup.sh
rename from taskcluster/docker/lint/Dockerfile rename to testing/docker/lint/Dockerfile --- a/taskcluster/docker/lint/Dockerfile +++ b/testing/docker/lint/Dockerfile @@ -1,30 +1,30 @@ FROM ubuntu:16.04 MAINTAINER Andrew Halberstadt <ahalberstadt@mozilla.com> RUN useradd -d /home/worker -s /bin/bash -m worker WORKDIR /home/worker RUN mkdir /build -# %include taskcluster/docker/recipes/tooltool.py -ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /build/tooltool.py +# %include testing/docker/recipes/tooltool.py +ADD topsrcdir/testing/docker/recipes/tooltool.py /build/tooltool.py # %include testing/mozharness/external_tools/robustcheckout.py ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py -# %include taskcluster/docker/recipes/install-mercurial.sh -ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /build/install-mercurial.sh +# %include testing/docker/recipes/install-mercurial.sh +ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /build/install-mercurial.sh ADD system-setup.sh /tmp/system-setup.sh # %include tools/lint/flake8/flake8_requirements.txt ADD topsrcdir/tools/lint/flake8/flake8_requirements.txt /tmp/flake8_requirements.txt RUN bash /tmp/system-setup.sh -# %include taskcluster/docker/recipes/run-task -ADD topsrcdir/taskcluster/docker/recipes/run-task /home/worker/bin/run-task +# %include testing/docker/recipes/run-task +ADD topsrcdir/testing/docker/recipes/run-task /home/worker/bin/run-task RUN chown -R worker:worker /home/worker/bin && chmod 755 /home/worker/bin/* # Set variable normally configured at login, by the shells parent process, these # are taken from GNU su manual ENV HOME /home/worker ENV SHELL /bin/bash ENV USER worker ENV LOGNAME worker
rename from taskcluster/docker/recipes/centos6-build-system-setup.sh rename to testing/docker/recipes/centos6-build-system-setup.sh
rename from taskcluster/docker/recipes/install-mercurial.sh rename to testing/docker/recipes/install-mercurial.sh
rename from taskcluster/docker/recipes/ubuntu1204-test-system-setup.sh rename to testing/docker/recipes/ubuntu1204-test-system-setup.sh
rename from taskcluster/docker/recipes/ubuntu1604-test-system-setup.sh rename to testing/docker/recipes/ubuntu1604-test-system-setup.sh
rename from taskcluster/docker/rust-build/build_cargo.sh rename to testing/docker/rust-build/build_cargo.sh
rename from taskcluster/docker/rust-build/build_rust.sh rename to testing/docker/rust-build/build_rust.sh
rename from taskcluster/docker/rust-build/build_rust_mac.sh rename to testing/docker/rust-build/build_rust_mac.sh
rename from taskcluster/docker/rust-build/fetch_cargo.sh rename to testing/docker/rust-build/fetch_cargo.sh
rename from taskcluster/docker/rust-build/fetch_rust.sh rename to testing/docker/rust-build/fetch_rust.sh
rename from taskcluster/docker/rust-build/package_rust.sh rename to testing/docker/rust-build/package_rust.sh
rename from taskcluster/docker/rust-build/repack_rust.py rename to testing/docker/rust-build/repack_rust.py --- a/taskcluster/docker/rust-build/repack_rust.py +++ b/testing/docker/rust-build/repack_rust.py @@ -5,183 +5,171 @@ with the necessary tool and target suppo build environment. ''' import os.path import requests import subprocess import toml - def fetch_file(url): - '''Download a file from the given url if it's not already present.''' - filename = os.path.basename(url) - if os.path.exists(filename): - return - r = requests.get(url, stream=True) - r.raise_for_status() - with open(filename, 'wb') as fd: - for chunk in r.iter_content(4096): - fd.write(chunk) - + '''Download a file from the given url if it's not already present.''' + filename = os.path.basename(url) + if os.path.exists(filename): + return + r = requests.get(url, stream=True) + r.raise_for_status() + with open(filename, 'wb') as fd: + for chunk in r.iter_content(4096): + fd.write(chunk) def fetch(url): - '''Download and verify a package url.''' - base = os.path.basename(url) - print('Fetching %s...' % base) - fetch_file(url + '.asc') - fetch_file(url) - fetch_file(url + '.sha256') - fetch_file(url + '.asc.sha256') - print('Verifying %s...' % base) - subprocess.check_call(['shasum', '-c', base + '.sha256']) - subprocess.check_call(['shasum', '-c', base + '.asc.sha256']) - subprocess.check_call(['gpg', '--verify', base + '.asc', base]) - if False: - subprocess.check_call([ - 'keybase', 'pgp', 'verify', '-d', base + '.asc', ' -i', base, - ]) - + '''Download and verify a package url.''' + base = os.path.basename(url) + print('Fetching %s...' % base) + fetch_file(url + '.asc') + fetch_file(url) + fetch_file(url + '.sha256') + fetch_file(url + '.asc.sha256') + print('Verifying %s...' % base) + subprocess.check_call(['shasum', '-c', base + '.sha256']) + subprocess.check_call(['shasum', '-c', base + '.asc.sha256']) + subprocess.check_call(['gpg', '--verify', base + '.asc', base]) + if False: + subprocess.check_call(['keybase', 'pgp', 'verify', + '-d', base + '.asc', + '-i', base, + ]) def install(filename, target): - '''Run a package's installer script against the given target directory.''' - print(' Unpacking %s...' % filename) - subprocess.check_call(['tar', 'xf', filename]) - basename = filename.split('.tar')[0] - print(' Installing %s...' % basename) - install_cmd = [os.path.join(basename, 'install.sh')] - install_cmd += ['--prefix=' + os.path.abspath(target)] - install_cmd += ['--disable-ldconfig'] - subprocess.check_call(install_cmd) - print(' Cleaning %s...' % basename) - subprocess.check_call(['rm', '-rf', basename]) - + '''Run a package's installer script against the given target directory.''' + print(' Unpacking %s...' % filename) + subprocess.check_call(['tar', 'xf', filename]) + basename = filename.split('.tar')[0] + print(' Installing %s...' % basename) + install_cmd = [os.path.join(basename, 'install.sh')] + install_cmd += ['--prefix=' + os.path.abspath(target)] + install_cmd += ['--disable-ldconfig'] + subprocess.check_call(install_cmd) + print(' Cleaning %s...' % basename) + subprocess.check_call(['rm', '-rf', basename]) def package(manifest, pkg, target): - '''Pull out the package dict for a particular package and target - from the given manifest.''' - version = manifest['pkg'][pkg]['version'] - info = manifest['pkg'][pkg]['target'][target] - return (version, info) - + '''Pull out the package dict for a particular package and target + from the given manifest.''' + version = manifest['pkg'][pkg]['version'] + info = manifest['pkg'][pkg]['target'][target] + return (version, info) def fetch_package(manifest, pkg, host): - version, info = package(manifest, pkg, host) - print('%s %s\n %s\n %s' % (pkg, version, info['url'], info['hash'])) - if not info['available']: - print('%s marked unavailable for %s' % (pkg, host)) - raise AssertionError - fetch(info['url']) - return info - + version, info = package(manifest, pkg, host) + print('%s %s\n %s\n %s' % (pkg, version, info['url'], info['hash'])) + if not info['available']: + print('%s marked unavailable for %s' % (pkg, host)) + raise AssertionError + fetch(info['url']) + return info def fetch_std(manifest, targets): - stds = [] - for target in targets: - info = fetch_package(manifest, 'rust-std', target) - stds.append(info) - return stds - + stds = [] + for target in targets: + info = fetch_package(manifest, 'rust-std', target) + stds.append(info) + return stds def tar_for_host(host): - if 'linux' in host: - tar_options = 'cJf' - tar_ext = '.tar.xz' - else: - tar_options = 'cjf' - tar_ext = '.tar.bz2' - return tar_options, tar_ext - + if 'linux' in host: + tar_options = 'cJf' + tar_ext = '.tar.xz' + else: + tar_options = 'cjf' + tar_ext = '.tar.bz2' + return tar_options, tar_ext def repack(host, targets, channel='stable', suffix=''): - print("Repacking rust for %s..." % host) - url = 'https://static.rust-lang.org/dist/channel-rust-' + channel + '.toml' - req = requests.get(url) - req.raise_for_status() - manifest = toml.loads(req.content) - if manifest['manifest-version'] != '2': - print('ERROR: unrecognized manifest version %s.' % manifest[ - 'manifest-version']) - return - print('Using manifest for rust %s as of %s.' % (channel, manifest['date'])) - print('Fetching packages...') - rustc = fetch_package(manifest, 'rustc', host) - cargo = fetch_package(manifest, 'cargo', host) - stds = fetch_std(manifest, targets) - print('Installing packages...') - tar_basename = 'rustc-' + host - if suffix: - tar_basename += '-' + suffix - tar_basename += '-repack' - install_dir = 'rustc' - subprocess.check_call(['rm', '-rf', install_dir]) - install(os.path.basename(rustc['url']), install_dir) - install(os.path.basename(cargo['url']), install_dir) - for std in stds: - install(os.path.basename(std['url']), install_dir) - pass - print('Tarring %s...' % tar_basename) - tar_options, tar_ext = tar_for_host(host) - subprocess.check_call( - ['tar', tar_options, tar_basename + tar_ext, install_dir]) - subprocess.check_call(['rm', '-rf', install_dir]) - + print("Repacking rust for %s..." % host) + url = 'https://static.rust-lang.org/dist/channel-rust-' + channel + '.toml' + req = requests.get(url) + req.raise_for_status() + manifest = toml.loads(req.content) + if manifest['manifest-version'] != '2': + print('ERROR: unrecognized manifest version %s.' % manifest['manifest-version']) + return + print('Using manifest for rust %s as of %s.' % (channel, manifest['date'])) + print('Fetching packages...') + rustc = fetch_package(manifest, 'rustc', host) + cargo = fetch_package(manifest, 'cargo', host) + stds = fetch_std(manifest, targets) + print('Installing packages...') + tar_basename = 'rustc-' + host + if suffix: + tar_basename += '-' + suffix + tar_basename += '-repack' + install_dir = 'rustc' + subprocess.check_call(['rm', '-rf', install_dir]) + install(os.path.basename(rustc['url']), install_dir) + install(os.path.basename(cargo['url']), install_dir) + for std in stds: + install(os.path.basename(std['url']), install_dir) + pass + print('Tarring %s...' % tar_basename) + tar_options, tar_ext = tar_for_host(host) + subprocess.check_call(['tar', tar_options, tar_basename + tar_ext, install_dir]) + subprocess.check_call(['rm', '-rf', install_dir]) def repack_cargo(host, channel='nightly'): - print("Repacking cargo for %s..." % host) - # Cargo doesn't seem to have a .toml manifest. - base_url = 'https://static.rust-lang.org/cargo-dist/' - req = requests.get(os.path.join(base_url, 'channel-cargo-' + channel)) - req.raise_for_status() - file = '' - for line in req.iter_lines(): - if line.find(host) != -1: - file = line.strip() - if not file: - print('No manifest entry for %s!' % host) - return - manifest = { - 'date': req.headers['Last-Modified'], - 'pkg': { - 'cargo': { - 'version': channel, - 'target': { - host: { - 'url': os.path.join(base_url, file), - 'hash': None, - 'available': True, - }, - }, - }, - }, - } - print('Using manifest for cargo %s.' % channel) - print('Fetching packages...') - cargo = fetch_package(manifest, 'cargo', host) - print('Installing packages...') - install_dir = 'cargo' - subprocess.check_call(['rm', '-rf', install_dir]) - install(os.path.basename(cargo['url']), install_dir) - tar_basename = 'cargo-%s-repack' % host - print('Tarring %s...' % tar_basename) - tar_options, tar_ext = tar_for_host(host) - subprocess.check_call( - ['tar', tar_options, tar_basename + tar_ext, install_dir]) - subprocess.check_call(['rm', '-rf', install_dir]) - + print("Repacking cargo for %s..." % host) + # Cargo doesn't seem to have a .toml manifest. + base_url = 'https://static.rust-lang.org/cargo-dist/' + req = requests.get(os.path.join(base_url, 'channel-cargo-' + channel)) + req.raise_for_status() + file = '' + for line in req.iter_lines(): + if line.find(host) != -1: + file = line.strip() + if not file: + print('No manifest entry for %s!' % host) + return + manifest = { + 'date': req.headers['Last-Modified'], + 'pkg': { + 'cargo': { + 'version': channel, + 'target': { + host: { + 'url': os.path.join(base_url, file), + 'hash': None, + 'available': True, + }, + }, + }, + }, + } + print('Using manifest for cargo %s.' % channel) + print('Fetching packages...') + cargo = fetch_package(manifest, 'cargo', host) + print('Installing packages...') + install_dir = 'cargo' + subprocess.check_call(['rm', '-rf', install_dir]) + install(os.path.basename(cargo['url']), install_dir) + tar_basename = 'cargo-%s-repack' % host + print('Tarring %s...' % tar_basename) + tar_options, tar_ext = tar_for_host(host) + subprocess.check_call(['tar', tar_options, tar_basename + tar_ext, install_dir]) + subprocess.check_call(['rm', '-rf', install_dir]) # rust platform triples -android = "armv7-linux-androideabi" -android_x86 = "i686-linux-android" -linux64 = "x86_64-unknown-linux-gnu" -linux32 = "i686-unknown-linux-gnu" -mac64 = "x86_64-apple-darwin" -mac32 = "i686-apple-darwin" -win64 = "x86_64-pc-windows-msvc" -win32 = "i686-pc-windows-msvc" +android="armv7-linux-androideabi" +android_x86="i686-linux-android" +linux64="x86_64-unknown-linux-gnu" +linux32="i686-unknown-linux-gnu" +mac64="x86_64-apple-darwin" +mac32="i686-apple-darwin" +win64="x86_64-pc-windows-msvc" +win32="i686-pc-windows-msvc" if __name__ == '__main__': - repack(mac64, [mac64, mac32]) - repack(win32, [win32]) - repack(win64, [win64]) - repack(linux64, [linux64, linux32]) - repack(linux64, [linux64, mac64, mac32], suffix='mac-cross') - repack(linux64, [linux64, android, android_x86], suffix='android-cross') + repack(mac64, [mac64, mac32]) + repack(win32, [win32]) + repack(win64, [win64]) + repack(linux64, [linux64, linux32]) + repack(linux64, [linux64, mac64, mac32], suffix='mac-cross') + repack(linux64, [linux64, android, android_x86], suffix='android-cross')
rename from taskcluster/docker/rust-build/splat_rust.py rename to testing/docker/rust-build/splat_rust.py --- a/taskcluster/docker/rust-build/splat_rust.py +++ b/testing/docker/rust-build/splat_rust.py @@ -5,47 +5,43 @@ tree to update them to a new set of rust ''' import json import os.path import sys from collections import OrderedDict - def load_manifest(path): with open(path) as f: return json.load(f, object_pairs_hook=OrderedDict) return None - def save_manifest(manifest, path): with open(path, 'w') as f: json.dump(manifest, f, - indent=0, separators=(',', ': ')) + indent=0, separators=(',', ': ')) f.write('\n') - def replace(manifest, stanza): key = 'rustc' version = stanza.get('version') for s in manifest: if key in s.get('filename'): if version: print('Replacing %s\n with %s' % (s['version'], version)) s['version'] = version print(' old %s' % s['digest'][:12]) s['digest'] = stanza['digest'] s['size'] = stanza['size'] print(' new %s' % s['digest'][:12]) return True - print('Warning: Could not find matching %s filename' % key) + print('Warning: Could not find matching %s filename' %key) return False - def update_manifest(source_manifest, target, target_filename): for stanza in source_manifest: filename = stanza.get('filename') if target in filename: size = int(stanza.get('size')) print('Found %s %d bytes' % (filename, size)) version = stanza.get('version') if version: @@ -98,9 +94,9 @@ if __name__ == '__main__': sys.exit(1) base_path = sys.argv[1] updates = load_manifest('manifest.tt') for target, filenames in TARGETS.items(): for target_filename in filenames: update_manifest(updates, target, - os.path.join(base_path, target_filename)) + os.path.join(base_path, target_filename))
rename from taskcluster/docker/rust-build/tcbuild.py rename to testing/docker/rust-build/tcbuild.py --- a/taskcluster/docker/rust-build/tcbuild.py +++ b/testing/docker/rust-build/tcbuild.py @@ -15,70 +15,60 @@ import json import os import shutil import sys import taskcluster import tempfile import time import tooltool -requests.packages.urllib3.disable_warnings() - - def local_file(filename): ''' Return a path to a file next to this script. ''' return os.path.join(os.path.dirname(__file__), filename) - def read_tc_auth(tc_auth_file): ''' Read taskcluster credentials from tc_auth_file and return them as a dict. ''' return json.load(open(tc_auth_file, 'rb')) - def fill_template_dict(d, keys): for key, val in d.items(): if isinstance(val, basestring) and '{' in val: d[key] = val.format(**keys) elif isinstance(val, dict): fill_template_dict(val, keys) - def fill_template(template_file, keys): ''' Take the file object template_file, parse it as JSON, and interpolate (using str.template) its keys using keys. ''' template = json.load(template_file) fill_template_dict(template, keys) return template - def spawn_task(queue, args): ''' Spawn a Taskcluster task in queue using args. ''' task_id = taskcluster.utils.slugId() with open(local_file('task.json'), 'rb') as template: keys = vars(args) now = datetime.datetime.utcnow() keys['task_created'] = now.isoformat() + 'Z' - keys['task_deadline'] = (now + datetime.timedelta( - hours=2)).isoformat() + 'Z' - keys['artifacts_expires'] = (now + datetime.timedelta( - days=1)).isoformat() + 'Z' + keys['task_deadline'] = (now + datetime.timedelta(hours=2)).isoformat() + 'Z' + keys['artifacts_expires'] = (now + datetime.timedelta(days=1)).isoformat() + 'Z' payload = fill_template(template, keys) queue.createTask(task_id, payload) print('--- %s task %s submitted ---' % (now, task_id)) return task_id - def wait_for_task(queue, task_id, initial_wait=5): ''' Wait until queue reports that task task_id is completed, and return its run id. Sleep for initial_wait seconds before checking status the first time. Then poll periodically and print a running log of the task status. ''' @@ -86,30 +76,29 @@ def wait_for_task(queue, task_id, initia previous_state = None have_ticks = False while True: res = queue.status(task_id) state = res['status']['state'] if state != previous_state: now = datetime.datetime.utcnow() if have_ticks: - sys.stdout.write('\n') - have_ticks = False + sys.stdout.write('\n') + have_ticks = False print('--- %s task %s %s ---' % (now, task_id, state)) previous_state = state if state == 'completed': return len(res['status']['runs']) - 1 if state in ('failed', 'exception'): raise Exception('Task failed') sys.stdout.write('.') sys.stdout.flush() have_ticks = True time.sleep(10) - def fetch_artifact(queue, task_id, run_id, name, dest_dir): ''' Fetch the artifact with name from task_id and run_id in queue, write it to a file in dest_dir, and return the path to the written file. ''' url = queue.buildUrl('getArtifact', task_id, run_id, name) fn = os.path.join(dest_dir, os.path.basename(name)) @@ -120,23 +109,21 @@ def fetch_artifact(queue, task_id, run_i with open(fn, 'wb') as f: for chunk in r.iter_content(1024): f.write(chunk) except requests.exceptions.HTTPError: print('HTTP Error %d fetching %s' % (r.status_code, name)) return None return fn - def make_artifact_dir(task_id, run_id): prefix = 'tc-artifacts.%s.%d.' % (task_id, run_id) print('making artifact dir %s' % prefix) return tempfile.mkdtemp(prefix=prefix) - def fetch_artifacts(queue, task_id, run_id): ''' Fetch all artifacts from task_id and run_id in queue, write them to temporary files, and yield the path to each. ''' try: tempdir = make_artifact_dir(task_id, run_id) res = queue.listArtifacts(task_id, run_id) @@ -145,21 +132,20 @@ def fetch_artifacts(queue, task_id, run_ if a['name'].startswith('public/logs'): continue # Skip interfaces if a['name'].startswith('private/docker-worker'): continue yield fetch_artifact(queue, task_id, run_id, a['name'], tempdir) finally: if os.path.isdir(tempdir): - # shutil.rmtree(tempdir) + #shutil.rmtree(tempdir) print('Artifacts downloaded to %s' % tempdir) pass - def upload_to_tooltool(tooltool_auth, task_id, artifact): ''' Upload artifact to tooltool using tooltool_auth as the authentication token. Return the path to the generated tooltool manifest. ''' try: oldcwd = os.getcwd() os.chdir(os.path.dirname(artifact)) @@ -177,54 +163,43 @@ def upload_to_tooltool(tooltool_auth, ta '-m', manifest, '--authentication-file', tooltool_auth, '--message', 'Built from taskcluster task {}'.format(task_id), ]) return manifest finally: os.chdir(oldcwd) - def update_manifest(artifact, manifest, local_gecko_clone): platform = 'linux' manifest_dir = os.path.join(local_gecko_clone, 'testing', 'config', 'tooltool-manifests') platform_dir = [p for p in os.listdir(manifest_dir) if p.startswith(platform)][0] tree_manifest = os.path.join(manifest_dir, platform_dir, 'releng.manifest') print('%s -> %s' % (manifest, tree_manifest)) shutil.copyfile(manifest, tree_manifest) - def main(): parser = argparse.ArgumentParser(description='Build and upload binaries') - parser.add_argument('taskcluster_auth', - help='Path to a file containing Taskcluster client ' - 'ID and authentication token as a JSON file in ' - 'the form {"clientId": "...", "accessToken": "..."}') - parser.add_argument('--tooltool-auth', - help='Path to a file containing a tooltool ' - 'authentication token valid for uploading files') - parser.add_argument('--local-gecko-clone', - help='Path to a local Gecko clone whose tooltool ' - 'manifests will be updated with the newly-built binaries') + parser.add_argument('taskcluster_auth', help='Path to a file containing Taskcluster client ID and authentication token as a JSON file in the form {"clientId": "...", "accessToken": "..."}') + parser.add_argument('--tooltool-auth', help='Path to a file containing a tooltool authentication token valid for uploading files') + parser.add_argument('--local-gecko-clone', help='Path to a local Gecko clone whose tooltool manifests will be updated with the newly-built binaries') parser.add_argument('--rust-branch', default='stable', help='Revision of the rust repository to use') parser.add_argument('--task', help='Use an existing task') args = parser.parse_args() tc_auth = read_tc_auth(args.taskcluster_auth) queue = taskcluster.Queue({'credentials': tc_auth}) if args.task: task_id, initial_wait = args.task, 0 else: task_id, initial_wait = spawn_task(queue, args), 25 run_id = wait_for_task(queue, task_id, initial_wait) for artifact in fetch_artifacts(queue, task_id, run_id): if args.tooltool_auth: - manifest = upload_to_tooltool(args.tooltool_auth, task_id, - artifact) + manifest = upload_to_tooltool(args.tooltool_auth, task_id, artifact) if args.local_gecko_clone: update_manifest(artifact, manifest, args.local_gecko_clone) - if __name__ == '__main__': main()
rename from taskcluster/docker/rust-build/upload_rust.sh rename to testing/docker/rust-build/upload_rust.sh
rename from taskcluster/docker/tester/dot-config/pip/pip.conf rename to testing/docker/tester/dot-config/pip/pip.conf
rename from taskcluster/docker/tester/dot-config/user-dirs.dirs rename to testing/docker/tester/dot-config/user-dirs.dirs
rename from taskcluster/docker/tester/dot-config/user-dirs.locale rename to testing/docker/tester/dot-config/user-dirs.locale
rename from taskcluster/docker/tester/dot-pulse/default.pa rename to testing/docker/tester/dot-pulse/default.pa
rename from taskcluster/docker/tester/tc-vcs-config.yml rename to testing/docker/tester/tc-vcs-config.yml
rename from taskcluster/docker/upload-symbols/Dockerfile rename to testing/docker/upload-symbols/Dockerfile
rename from taskcluster/docker/upload-symbols/README.md rename to testing/docker/upload-symbols/README.md
rename from taskcluster/docker/upload-symbols/bin/checkout-script.sh rename to testing/docker/upload-symbols/bin/checkout-script.sh