--- a/AUTHORS
+++ b/AUTHORS
@@ -874,16 +874,17 @@ Roy Yokoyama <yokoyama@netscape.com>
RSA Security, Inc
Russell King <rmk@arm.linux.org.uk>
Rusty Lynch <rusty.lynch@intel.com>
Ryan Cassin <rcassin@supernova.org>
Ryan Flint <rflint@dslr.net>
Ryan Jones <sciguyryan@gmail.com>
Ryan VanderMeulen <ryanvm@gmail.com>
Ryoichi Furukawa <oliver@1000cp.com>
+Sanyam Khurana <Sanyam.Khurana01@gmail.com>
sagdjb@softwareag.com
Samir Gehani <sgehani@netscape.com>
Sammy Ford
Samphan Raruenrom
Samuel Sieb <samuel@sieb.net>
Sarlos Tamas
scole@planetweb.com
Scooter Morris <scootermorris@comcast.net>
--- a/taskcluster/ci/android-stuff/kind.yml
+++ b/taskcluster/ci/android-stuff/kind.yml
@@ -58,17 +58,17 @@ jobs:
- "/home/worker/bin/before.sh && /home/worker/bin/build.sh && /home/worker/bin/after.sh && true\n"
max-run-time: 36000
scopes:
- docker-worker:relengapi-proxy:tooltool.download.internal
- docker-worker:relengapi-proxy:tooltool.download.public
when:
files-changed:
- "mobile/android/config/**"
- - "testing/docker/android-gradle-build/**"
+ - "taskcluster/docker/android-gradle-build/**"
- "testing/mozharness/configs/builds/releng_sub_android_configs/*gradle_dependencies.py"
- "**/*.gradle"
android-test:
description: "Android armv7 unit tests"
attributes:
build_platform: android-test
build_type: opt
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -1,14 +1,14 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
implementation: 'taskgraph.task.docker_image:DockerImageTask'
-images_path: '../../../testing/docker'
+images_path: '../../../taskcluster/docker'
# make a task for each docker-image we might want. For the moment, since we
# write artifacts for each, these are whitelisted, but ideally that will change
# (to use subdirectory clones of the proper directory), at which point we can
# generate tasks for every docker image in the directory, secure in the
# knowledge that unnecessary images will be omitted from the target task graph
images:
desktop-test: dt
--- a/taskcluster/ci/source-check/mozlint.yml
+++ b/taskcluster/ci/source-check/mozlint.yml
@@ -35,17 +35,17 @@ mozlint-eslint/opt:
# Run when eslint policies change.
- '**/.eslintignore'
- '**/*eslintrc*'
# The plugin implementing custom checks.
- 'tools/lint/eslint/eslint-plugin-mozilla/**'
# Other misc lint related files.
- 'python/mozlint/**'
- 'tools/lint/**'
- - 'testing/docker/lint/**'
+ - 'taskcluster/docker/lint/**'
mozlint-flake8/opt:
description: flake8 run over the gecko codebase
treeherder:
symbol: f8
kind: test
tier: 1
platform: lint/opt
@@ -61,17 +61,17 @@ mozlint-flake8/opt:
- integration
- release
when:
files-changed:
- '**/*.py'
- '**/.flake8'
- 'python/mozlint/**'
- 'tools/lint/**'
- - 'testing/docker/lint/**'
+ - 'taskcluster/docker/lint/**'
wptlint-gecko/opt:
description: web-platform-tests linter
treeherder:
symbol: W
kind: test
tier: 1
platform: lint/opt
@@ -89,9 +89,9 @@ wptlint-gecko/opt:
when:
files-changed:
- 'testing/web-platform/tests/**'
- 'testing/web-platform/mozilla/tests/**'
- 'testing/web-platform/meta/MANIFEST.json'
- 'testing/web-platform/mozilla/meta/MANIFEST.json'
- 'python/mozlint/**'
- 'tools/lint/**'
- - 'testing/docker/lint/**'
+ - 'taskcluster/docker/lint/**'
rename from testing/docker/README.md
rename to taskcluster/docker/README.md
rename from testing/docker/REGISTRY
rename to taskcluster/docker/REGISTRY
rename from testing/docker/android-gradle-build/Dockerfile
rename to taskcluster/docker/android-gradle-build/Dockerfile
--- a/testing/docker/android-gradle-build/Dockerfile
+++ b/taskcluster/docker/android-gradle-build/Dockerfile
@@ -9,18 +9,18 @@ VOLUME /home/worker/workspace
VOLUME /home/worker/tooltool-cache
# Add build scripts; these are the entry points from the taskcluster worker, and
# operate on environment variables
ADD bin /home/worker/bin
RUN chmod +x /home/worker/bin/*
# Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb
-# %include testing/docker/recipes/xvfb.sh
-ADD topsrcdir/testing/docker/recipes/xvfb.sh /home/worker/scripts/xvfb.sh
+# %include taskcluster/docker/recipes/xvfb.sh
+ADD topsrcdir/taskcluster/docker/recipes/xvfb.sh /home/worker/scripts/xvfb.sh
# Add configuration
COPY dot-config /home/worker/.config
# Generate machine uuid file
RUN dbus-uuidgen --ensure=/var/lib/dbus/machine-id
# Stubbed out credentials; mozharness looks for this file an issues a WARNING
rename from testing/docker/android-gradle-build/README.md
rename to taskcluster/docker/android-gradle-build/README.md
rename from testing/docker/android-gradle-build/REGISTRY
rename to taskcluster/docker/android-gradle-build/REGISTRY
rename from testing/docker/android-gradle-build/VERSION
rename to taskcluster/docker/android-gradle-build/VERSION
rename from testing/docker/android-gradle-build/bin/after.sh
rename to taskcluster/docker/android-gradle-build/bin/after.sh
rename from testing/docker/android-gradle-build/bin/before.sh
rename to taskcluster/docker/android-gradle-build/bin/before.sh
rename from testing/docker/android-gradle-build/bin/build.sh
rename to taskcluster/docker/android-gradle-build/bin/build.sh
rename from testing/docker/android-gradle-build/bin/checkout-script.sh
rename to taskcluster/docker/android-gradle-build/bin/checkout-script.sh
rename from testing/docker/android-gradle-build/bin/checkout-sources.sh
rename to taskcluster/docker/android-gradle-build/bin/checkout-sources.sh
rename from testing/docker/android-gradle-build/buildprops.json
rename to taskcluster/docker/android-gradle-build/buildprops.json
rename from testing/docker/android-gradle-build/dot-config/pip/pip.conf
rename to taskcluster/docker/android-gradle-build/dot-config/pip/pip.conf
rename from testing/docker/android-gradle-build/oauth.txt
rename to taskcluster/docker/android-gradle-build/oauth.txt
rename from testing/docker/base-build/Dockerfile
rename to taskcluster/docker/base-build/Dockerfile
rename from testing/docker/base-build/VERSION
rename to taskcluster/docker/base-build/VERSION
rename from testing/docker/base-build/system-setup.sh
rename to taskcluster/docker/base-build/system-setup.sh
rename from testing/docker/base-test/Dockerfile
rename to taskcluster/docker/base-test/Dockerfile
rename from testing/docker/base-test/REGISTRY
rename to taskcluster/docker/base-test/REGISTRY
rename from testing/docker/base-test/VERSION
rename to taskcluster/docker/base-test/VERSION
rename from testing/docker/base-test/sources.list
rename to taskcluster/docker/base-test/sources.list
rename from testing/docker/centos6-build-upd/Dockerfile
rename to taskcluster/docker/centos6-build-upd/Dockerfile
rename from testing/docker/centos6-build-upd/REGISTRY
rename to taskcluster/docker/centos6-build-upd/REGISTRY
rename from testing/docker/centos6-build-upd/VERSION
rename to taskcluster/docker/centos6-build-upd/VERSION
rename from testing/docker/centos6-build/Dockerfile
rename to taskcluster/docker/centos6-build/Dockerfile
rename from testing/docker/centos6-build/REGISTRY
rename to taskcluster/docker/centos6-build/REGISTRY
rename from testing/docker/centos6-build/VERSION
rename to taskcluster/docker/centos6-build/VERSION
rename from testing/docker/centos6-build/hgrc
rename to taskcluster/docker/centos6-build/hgrc
rename from testing/docker/centos6-build/system-setup.sh
rename to taskcluster/docker/centos6-build/system-setup.sh
rename from testing/docker/decision/Dockerfile
rename to taskcluster/docker/decision/Dockerfile
--- a/testing/docker/decision/Dockerfile
+++ b/taskcluster/docker/decision/Dockerfile
@@ -1,28 +1,28 @@
FROM ubuntu:16.04
MAINTAINER Greg Arndt <garndt@mozilla.com>
# Add worker user
RUN useradd -d /home/worker -s /bin/bash -m worker
RUN mkdir /home/worker/artifacts && chown worker:worker /home/worker/artifacts
-# %include testing/docker/recipes/tooltool.py
-ADD topsrcdir/testing/docker/recipes/tooltool.py /tmp/tooltool.py
+# %include taskcluster/docker/recipes/tooltool.py
+ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /tmp/tooltool.py
# %include testing/mozharness/external_tools/robustcheckout.py
ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
-# %include testing/docker/recipes/install-mercurial.sh
-ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /tmp/install-mercurial.sh
+# %include taskcluster/docker/recipes/install-mercurial.sh
+ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /tmp/install-mercurial.sh
ADD system-setup.sh /tmp/system-setup.sh
RUN bash /tmp/system-setup.sh
-# %include testing/docker/recipes/run-task
-ADD topsrcdir/testing/docker/recipes/run-task /home/worker/bin/run-task
+# %include taskcluster/docker/recipes/run-task
+ADD topsrcdir/taskcluster/docker/recipes/run-task /home/worker/bin/run-task
ENV PATH /home/worker/bin:$PATH
ENV SHELL /bin/bash
ENV HOME /home/worker
# Set a default command useful for debugging
CMD ["/bin/bash", "--login"]
rename from testing/docker/decision/README.md
rename to taskcluster/docker/decision/README.md
rename from testing/docker/decision/REGISTRY
rename to taskcluster/docker/decision/REGISTRY
rename from testing/docker/decision/VERSION
rename to taskcluster/docker/decision/VERSION
rename from testing/docker/decision/system-setup.sh
rename to taskcluster/docker/decision/system-setup.sh
rename from testing/docker/desktop-build/Dockerfile
rename to taskcluster/docker/desktop-build/Dockerfile
--- a/testing/docker/desktop-build/Dockerfile
+++ b/taskcluster/docker/desktop-build/Dockerfile
@@ -6,43 +6,43 @@ MAINTAINER Dustin J. Mitchell <dustin
VOLUME /home/worker/workspace
VOLUME /home/worker/tooltool-cache
# Add build scripts; these are the entry points from the taskcluster worker, and
# operate on environment variables
ADD bin /home/worker/bin
RUN chmod +x /home/worker/bin/*
-# %include testing/docker/recipes/tooltool.py
-ADD topsrcdir/testing/docker/recipes/tooltool.py /builds/tooltool.py
-ADD topsrcdir/testing/docker/recipes/tooltool.py /setup/tooltool.py
+# %include taskcluster/docker/recipes/tooltool.py
+ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /builds/tooltool.py
+ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /setup/tooltool.py
# %include testing/mozharness/external_tools/robustcheckout.py
ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
-# %include testing/docker/recipes/common.sh
-ADD topsrcdir/testing/docker/recipes/common.sh /setup/common.sh
+# %include taskcluster/docker/recipes/common.sh
+ADD topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh
-# %include testing/docker/recipes/install-mercurial.sh
-ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh
+# %include taskcluster/docker/recipes/install-mercurial.sh
+ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh
-# %include testing/docker/recipes/centos6-build-system-setup.sh
-ADD topsrcdir/testing/docker/recipes/centos6-build-system-setup.sh /setup/system-setup.sh
+# %include taskcluster/docker/recipes/centos6-build-system-setup.sh
+ADD topsrcdir/taskcluster/docker/recipes/centos6-build-system-setup.sh /setup/system-setup.sh
# TODO remove once base image doesn't install Mercurial
RUN pip uninstall -y Mercurial
RUN bash /setup/system-setup.sh
# Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb
-# %include testing/docker/recipes/xvfb.sh
-ADD topsrcdir/testing/docker/recipes/xvfb.sh /home/worker/scripts/xvfb.sh
+# %include taskcluster/docker/recipes/xvfb.sh
+ADD topsrcdir/taskcluster/docker/recipes/xvfb.sh /home/worker/scripts/xvfb.sh
-# %include testing/docker/recipes/run-task
-ADD topsrcdir/testing/docker/recipes/run-task /home/worker/bin/run-task
+# %include taskcluster/docker/recipes/run-task
+ADD topsrcdir/taskcluster/docker/recipes/run-task /home/worker/bin/run-task
# Add configuration
COPY dot-config /home/worker/.config
# Generate machine uuid file
RUN dbus-uuidgen --ensure=/var/lib/dbus/machine-id
# Stubbed out credentials; mozharness looks for this file an issues a WARNING
rename from testing/docker/desktop-build/bin/build.sh
rename to taskcluster/docker/desktop-build/bin/build.sh
rename from testing/docker/desktop-build/bin/checkout-script.sh
rename to taskcluster/docker/desktop-build/bin/checkout-script.sh
rename from testing/docker/desktop-build/bin/checkout-sources.sh
rename to taskcluster/docker/desktop-build/bin/checkout-sources.sh
rename from testing/docker/desktop-build/buildprops.json
rename to taskcluster/docker/desktop-build/buildprops.json
rename from testing/docker/desktop-build/dot-config/pip/pip.conf
rename to taskcluster/docker/desktop-build/dot-config/pip/pip.conf
rename from testing/docker/desktop-build/oauth.txt
rename to taskcluster/docker/desktop-build/oauth.txt
rename from testing/docker/desktop-test/Dockerfile
rename to taskcluster/docker/desktop-test/Dockerfile
--- a/testing/docker/desktop-test/Dockerfile
+++ b/taskcluster/docker/desktop-test/Dockerfile
@@ -1,37 +1,37 @@
FROM ubuntu:12.04
MAINTAINER Jonas Finnemann Jensen <jopsen@gmail.com>
RUN useradd -d /home/worker -s /bin/bash -m worker
WORKDIR /home/worker
-# %include testing/docker/recipes/tooltool.py
-ADD topsrcdir/testing/docker/recipes/tooltool.py /setup/tooltool.py
+# %include taskcluster/docker/recipes/tooltool.py
+ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /setup/tooltool.py
# %include testing/mozharness/external_tools/robustcheckout.py
ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
-# %include testing/docker/recipes/install-mercurial.sh
-ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /tmp/install-mercurial.sh
+# %include taskcluster/docker/recipes/install-mercurial.sh
+ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /tmp/install-mercurial.sh
# Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb
-# %include testing/docker/recipes/xvfb.sh
-ADD topsrcdir/testing/docker/recipes/xvfb.sh /home/worker/scripts/xvfb.sh
+# %include taskcluster/docker/recipes/xvfb.sh
+ADD topsrcdir/taskcluster/docker/recipes/xvfb.sh /home/worker/scripts/xvfb.sh
# Add the tooltool manifest containing the minidump_stackwalk binary.
# %include testing/config/tooltool-manifests/linux64/releng.manifest
ADD topsrcdir/testing/config/tooltool-manifests/linux64/releng.manifest /tmp/minidump_stackwalk.manifest
-# %include testing/docker/recipes/ubuntu1204-test-system-setup.sh
-ADD topsrcdir/testing/docker/recipes/ubuntu1204-test-system-setup.sh /setup/system-setup.sh
+# %include taskcluster/docker/recipes/ubuntu1204-test-system-setup.sh
+ADD topsrcdir/taskcluster/docker/recipes/ubuntu1204-test-system-setup.sh /setup/system-setup.sh
RUN bash /setup/system-setup.sh
-# %include testing/docker/recipes/run-task
-ADD topsrcdir/testing/docker/recipes/run-task /home/worker/bin/run-task
+# %include taskcluster/docker/recipes/run-task
+ADD topsrcdir/taskcluster/docker/recipes/run-task /home/worker/bin/run-task
# %include taskcluster/scripts/tester/test-ubuntu.sh
ADD topsrcdir/taskcluster/scripts/tester/test-ubuntu.sh /home/worker/bin/test-linux.sh
# This will create a host mounted filesystem when the cache is stripped
# on Try. This cancels out some of the performance losses of aufs. See
# bug 1291940.
VOLUME /home/worker/checkouts
rename from testing/docker/desktop-test/apport
rename to taskcluster/docker/desktop-test/apport
rename from testing/docker/desktop-test/buildprops.json
rename to taskcluster/docker/desktop-test/buildprops.json
rename from testing/docker/desktop-test/deja-dup-monitor.desktop
rename to taskcluster/docker/desktop-test/deja-dup-monitor.desktop
rename from testing/docker/desktop-test/dot-files/config/pip/pip.conf
rename to taskcluster/docker/desktop-test/dot-files/config/pip/pip.conf
rename from testing/docker/desktop-test/dot-files/config/user-dirs.dirs
rename to taskcluster/docker/desktop-test/dot-files/config/user-dirs.dirs
rename from testing/docker/desktop-test/dot-files/config/user-dirs.locale
rename to taskcluster/docker/desktop-test/dot-files/config/user-dirs.locale
rename from testing/docker/desktop-test/dot-files/pulse/default.pa
rename to taskcluster/docker/desktop-test/dot-files/pulse/default.pa
rename from testing/docker/desktop-test/fonts.conf
rename to taskcluster/docker/desktop-test/fonts.conf
rename from testing/docker/desktop-test/jockey-gtk.desktop
rename to taskcluster/docker/desktop-test/jockey-gtk.desktop
rename from testing/docker/desktop-test/motd
rename to taskcluster/docker/desktop-test/motd
rename from testing/docker/desktop-test/release-upgrades
rename to taskcluster/docker/desktop-test/release-upgrades
rename from testing/docker/desktop-test/taskcluster-interactive-shell
rename to taskcluster/docker/desktop-test/taskcluster-interactive-shell
rename from testing/docker/desktop-test/tc-vcs-config.yml
rename to taskcluster/docker/desktop-test/tc-vcs-config.yml
rename from testing/docker/desktop-test/tester.env
rename to taskcluster/docker/desktop-test/tester.env
rename from testing/docker/desktop1604-test/Dockerfile
rename to taskcluster/docker/desktop1604-test/Dockerfile
--- a/testing/docker/desktop1604-test/Dockerfile
+++ b/taskcluster/docker/desktop1604-test/Dockerfile
@@ -1,40 +1,40 @@
FROM ubuntu:16.04
MAINTAINER Joel Maher <joel.maher@gmail.com>
RUN useradd -d /home/worker -s /bin/bash -m worker
WORKDIR /home/worker
-# %include testing/docker/recipes/tooltool.py
-ADD topsrcdir/testing/docker/recipes/tooltool.py /setup/tooltool.py
+# %include taskcluster/docker/recipes/tooltool.py
+ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /setup/tooltool.py
# %include testing/mozharness/external_tools/robustcheckout.py
ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
-# %include testing/docker/recipes/common.sh
-ADD topsrcdir/testing/docker/recipes/common.sh /setup/common.sh
+# %include taskcluster/docker/recipes/common.sh
+ADD topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh
-# %include testing/docker/recipes/install-mercurial.sh
-ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh
+# %include taskcluster/docker/recipes/install-mercurial.sh
+ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh
# Add the tooltool manifest containing the minidump_stackwalk binary.
# %include testing/config/tooltool-manifests/linux64/releng.manifest
ADD topsrcdir/testing/config/tooltool-manifests/linux64/releng.manifest /tmp/minidump_stackwalk.manifest
-# %include testing/docker/recipes/ubuntu1604-test-system-setup.sh
-ADD topsrcdir/testing/docker/recipes/ubuntu1604-test-system-setup.sh /setup/system-setup.sh
+# %include taskcluster/docker/recipes/ubuntu1604-test-system-setup.sh
+ADD topsrcdir/taskcluster/docker/recipes/ubuntu1604-test-system-setup.sh /setup/system-setup.sh
RUN bash /setup/system-setup.sh
# Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb
-# %include testing/docker/recipes/xvfb.sh
-ADD topsrcdir/testing/docker/recipes/xvfb.sh /home/worker/scripts/xvfb.sh
+# %include taskcluster/docker/recipes/xvfb.sh
+ADD topsrcdir/taskcluster/docker/recipes/xvfb.sh /home/worker/scripts/xvfb.sh
-# %include testing/docker/recipes/run-task
-ADD topsrcdir/testing/docker/recipes/run-task /home/worker/bin/run-task
+# %include taskcluster/docker/recipes/run-task
+ADD topsrcdir/taskcluster/docker/recipes/run-task /home/worker/bin/run-task
# %include taskcluster/scripts/tester/test-ubuntu.sh
ADD topsrcdir/taskcluster/scripts/tester/test-ubuntu.sh /home/worker/bin/test-linux.sh
# This will create a host mounted filesystem when the cache is stripped
# on Try. This cancels out some of the performance losses of aufs. See
# bug 1291940.
VOLUME /home/worker/checkouts
rename from testing/docker/desktop1604-test/apport
rename to taskcluster/docker/desktop1604-test/apport
rename from testing/docker/desktop1604-test/buildprops.json
rename to taskcluster/docker/desktop1604-test/buildprops.json
rename from testing/docker/desktop1604-test/deja-dup-monitor.desktop
rename to taskcluster/docker/desktop1604-test/deja-dup-monitor.desktop
rename from testing/docker/desktop1604-test/dot-files/config/pip/pip.conf
rename to taskcluster/docker/desktop1604-test/dot-files/config/pip/pip.conf
rename from testing/docker/desktop1604-test/dot-files/config/user-dirs.dirs
rename to taskcluster/docker/desktop1604-test/dot-files/config/user-dirs.dirs
rename from testing/docker/desktop1604-test/dot-files/config/user-dirs.locale
rename to taskcluster/docker/desktop1604-test/dot-files/config/user-dirs.locale
rename from testing/docker/desktop1604-test/dot-files/pulse/default.pa
rename to taskcluster/docker/desktop1604-test/dot-files/pulse/default.pa
rename from testing/docker/desktop1604-test/fonts.conf
rename to taskcluster/docker/desktop1604-test/fonts.conf
rename from testing/docker/desktop1604-test/jockey-gtk.desktop
rename to taskcluster/docker/desktop1604-test/jockey-gtk.desktop
rename from testing/docker/desktop1604-test/motd
rename to taskcluster/docker/desktop1604-test/motd
rename from testing/docker/desktop1604-test/release-upgrades
rename to taskcluster/docker/desktop1604-test/release-upgrades
rename from testing/docker/desktop1604-test/taskcluster-interactive-shell
rename to taskcluster/docker/desktop1604-test/taskcluster-interactive-shell
rename from testing/docker/desktop1604-test/tc-vcs-config.yml
rename to taskcluster/docker/desktop1604-test/tc-vcs-config.yml
rename from testing/docker/desktop1604-test/tester.env
rename to taskcluster/docker/desktop1604-test/tester.env
rename from testing/docker/image_builder/Dockerfile
rename to taskcluster/docker/image_builder/Dockerfile
--- a/testing/docker/image_builder/Dockerfile
+++ b/taskcluster/docker/image_builder/Dockerfile
@@ -1,24 +1,24 @@
FROM ubuntu:16.04
-# %include testing/docker/recipes/tooltool.py
-ADD topsrcdir/testing/docker/recipes/tooltool.py /setup/tooltool.py
+# %include taskcluster/docker/recipes/tooltool.py
+ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /setup/tooltool.py
-# %include testing/docker/recipes/common.sh
-ADD topsrcdir/testing/docker/recipes/common.sh /setup/common.sh
+# %include taskcluster/docker/recipes/common.sh
+ADD topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh
-# %include testing/docker/recipes/install-mercurial.sh
-ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh
+# %include taskcluster/docker/recipes/install-mercurial.sh
+ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh
-# %include testing/mozharness/external_tools/robustcheckout.py
-ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
+# %include taskcluster/mozharness/external_tools/robustcheckout.py
+ADD topsrcdir/taskcluster/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
-# %include testing/docker/recipes/run-task
-ADD topsrcdir/testing/docker/recipes/run-task /usr/local/bin/run-task
+# %include taskcluster/docker/recipes/run-task
+ADD topsrcdir/taskcluster/docker/recipes/run-task /usr/local/bin/run-task
# Add and run setup script
ADD build-image.sh /usr/local/bin/build-image.sh
ADD setup.sh /setup/setup.sh
RUN bash /setup/setup.sh
# Setup a workspace that won't use AUFS
VOLUME /home/worker/workspace
rename from testing/docker/image_builder/REGISTRY
rename to taskcluster/docker/image_builder/REGISTRY
rename from testing/docker/image_builder/VERSION
rename to taskcluster/docker/image_builder/VERSION
rename from testing/docker/image_builder/build-image.sh
rename to taskcluster/docker/image_builder/build-image.sh
rename from testing/docker/image_builder/setup.sh
rename to taskcluster/docker/image_builder/setup.sh
rename from testing/docker/lint/Dockerfile
rename to taskcluster/docker/lint/Dockerfile
--- a/testing/docker/lint/Dockerfile
+++ b/taskcluster/docker/lint/Dockerfile
@@ -1,30 +1,30 @@
FROM ubuntu:16.04
MAINTAINER Andrew Halberstadt <ahalberstadt@mozilla.com>
RUN useradd -d /home/worker -s /bin/bash -m worker
WORKDIR /home/worker
RUN mkdir /build
-# %include testing/docker/recipes/tooltool.py
-ADD topsrcdir/testing/docker/recipes/tooltool.py /build/tooltool.py
+# %include taskcluster/docker/recipes/tooltool.py
+ADD topsrcdir/taskcluster/docker/recipes/tooltool.py /build/tooltool.py
# %include testing/mozharness/external_tools/robustcheckout.py
ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
-# %include testing/docker/recipes/install-mercurial.sh
-ADD topsrcdir/testing/docker/recipes/install-mercurial.sh /build/install-mercurial.sh
+# %include taskcluster/docker/recipes/install-mercurial.sh
+ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /build/install-mercurial.sh
ADD system-setup.sh /tmp/system-setup.sh
# %include tools/lint/flake8/flake8_requirements.txt
ADD topsrcdir/tools/lint/flake8/flake8_requirements.txt /tmp/flake8_requirements.txt
RUN bash /tmp/system-setup.sh
-# %include testing/docker/recipes/run-task
-ADD topsrcdir/testing/docker/recipes/run-task /home/worker/bin/run-task
+# %include taskcluster/docker/recipes/run-task
+ADD topsrcdir/taskcluster/docker/recipes/run-task /home/worker/bin/run-task
RUN chown -R worker:worker /home/worker/bin && chmod 755 /home/worker/bin/*
# Set variable normally configured at login, by the shells parent process, these
# are taken from GNU su manual
ENV HOME /home/worker
ENV SHELL /bin/bash
ENV USER worker
ENV LOGNAME worker
rename from testing/docker/lint/system-setup.sh
rename to taskcluster/docker/lint/system-setup.sh
rename from testing/docker/recipes/centos6-build-system-setup.sh
rename to taskcluster/docker/recipes/centos6-build-system-setup.sh
rename from testing/docker/recipes/common.sh
rename to taskcluster/docker/recipes/common.sh
rename from testing/docker/recipes/install-mercurial.sh
rename to taskcluster/docker/recipes/install-mercurial.sh
rename from testing/docker/recipes/run-task
rename to taskcluster/docker/recipes/run-task
rename from testing/docker/recipes/tooltool.py
rename to taskcluster/docker/recipes/tooltool.py
rename from testing/docker/recipes/ubuntu1204-test-system-setup.sh
rename to taskcluster/docker/recipes/ubuntu1204-test-system-setup.sh
rename from testing/docker/recipes/ubuntu1604-test-system-setup.sh
rename to taskcluster/docker/recipes/ubuntu1604-test-system-setup.sh
rename from testing/docker/recipes/xvfb.sh
rename to taskcluster/docker/recipes/xvfb.sh
rename from testing/docker/rust-build/Dockerfile
rename to taskcluster/docker/rust-build/Dockerfile
rename from testing/docker/rust-build/README.md
rename to taskcluster/docker/rust-build/README.md
rename from testing/docker/rust-build/REGISTRY
rename to taskcluster/docker/rust-build/REGISTRY
rename from testing/docker/rust-build/VERSION
rename to taskcluster/docker/rust-build/VERSION
rename from testing/docker/rust-build/build_cargo.sh
rename to taskcluster/docker/rust-build/build_cargo.sh
rename from testing/docker/rust-build/build_rust.sh
rename to taskcluster/docker/rust-build/build_rust.sh
rename from testing/docker/rust-build/build_rust_mac.sh
rename to taskcluster/docker/rust-build/build_rust_mac.sh
rename from testing/docker/rust-build/fetch_cargo.sh
rename to taskcluster/docker/rust-build/fetch_cargo.sh
rename from testing/docker/rust-build/fetch_rust.sh
rename to taskcluster/docker/rust-build/fetch_rust.sh
rename from testing/docker/rust-build/package_rust.sh
rename to taskcluster/docker/rust-build/package_rust.sh
rename from testing/docker/rust-build/repack_rust.py
rename to taskcluster/docker/rust-build/repack_rust.py
--- a/testing/docker/rust-build/repack_rust.py
+++ b/taskcluster/docker/rust-build/repack_rust.py
@@ -5,171 +5,183 @@ with the necessary tool and target suppo
build environment.
'''
import os.path
import requests
import subprocess
import toml
+
def fetch_file(url):
- '''Download a file from the given url if it's not already present.'''
- filename = os.path.basename(url)
- if os.path.exists(filename):
- return
- r = requests.get(url, stream=True)
- r.raise_for_status()
- with open(filename, 'wb') as fd:
- for chunk in r.iter_content(4096):
- fd.write(chunk)
+ '''Download a file from the given url if it's not already present.'''
+ filename = os.path.basename(url)
+ if os.path.exists(filename):
+ return
+ r = requests.get(url, stream=True)
+ r.raise_for_status()
+ with open(filename, 'wb') as fd:
+ for chunk in r.iter_content(4096):
+ fd.write(chunk)
+
def fetch(url):
- '''Download and verify a package url.'''
- base = os.path.basename(url)
- print('Fetching %s...' % base)
- fetch_file(url + '.asc')
- fetch_file(url)
- fetch_file(url + '.sha256')
- fetch_file(url + '.asc.sha256')
- print('Verifying %s...' % base)
- subprocess.check_call(['shasum', '-c', base + '.sha256'])
- subprocess.check_call(['shasum', '-c', base + '.asc.sha256'])
- subprocess.check_call(['gpg', '--verify', base + '.asc', base])
- if False:
- subprocess.check_call(['keybase', 'pgp', 'verify',
- '-d', base + '.asc',
- '-i', base,
- ])
+ '''Download and verify a package url.'''
+ base = os.path.basename(url)
+ print('Fetching %s...' % base)
+ fetch_file(url + '.asc')
+ fetch_file(url)
+ fetch_file(url + '.sha256')
+ fetch_file(url + '.asc.sha256')
+ print('Verifying %s...' % base)
+ subprocess.check_call(['shasum', '-c', base + '.sha256'])
+ subprocess.check_call(['shasum', '-c', base + '.asc.sha256'])
+ subprocess.check_call(['gpg', '--verify', base + '.asc', base])
+ if False:
+ subprocess.check_call([
+ 'keybase', 'pgp', 'verify', '-d', base + '.asc', ' -i', base,
+ ])
+
def install(filename, target):
- '''Run a package's installer script against the given target directory.'''
- print(' Unpacking %s...' % filename)
- subprocess.check_call(['tar', 'xf', filename])
- basename = filename.split('.tar')[0]
- print(' Installing %s...' % basename)
- install_cmd = [os.path.join(basename, 'install.sh')]
- install_cmd += ['--prefix=' + os.path.abspath(target)]
- install_cmd += ['--disable-ldconfig']
- subprocess.check_call(install_cmd)
- print(' Cleaning %s...' % basename)
- subprocess.check_call(['rm', '-rf', basename])
+ '''Run a package's installer script against the given target directory.'''
+ print(' Unpacking %s...' % filename)
+ subprocess.check_call(['tar', 'xf', filename])
+ basename = filename.split('.tar')[0]
+ print(' Installing %s...' % basename)
+ install_cmd = [os.path.join(basename, 'install.sh')]
+ install_cmd += ['--prefix=' + os.path.abspath(target)]
+ install_cmd += ['--disable-ldconfig']
+ subprocess.check_call(install_cmd)
+ print(' Cleaning %s...' % basename)
+ subprocess.check_call(['rm', '-rf', basename])
+
def package(manifest, pkg, target):
- '''Pull out the package dict for a particular package and target
- from the given manifest.'''
- version = manifest['pkg'][pkg]['version']
- info = manifest['pkg'][pkg]['target'][target]
- return (version, info)
+ '''Pull out the package dict for a particular package and target
+ from the given manifest.'''
+ version = manifest['pkg'][pkg]['version']
+ info = manifest['pkg'][pkg]['target'][target]
+ return (version, info)
+
def fetch_package(manifest, pkg, host):
- version, info = package(manifest, pkg, host)
- print('%s %s\n %s\n %s' % (pkg, version, info['url'], info['hash']))
- if not info['available']:
- print('%s marked unavailable for %s' % (pkg, host))
- raise AssertionError
- fetch(info['url'])
- return info
+ version, info = package(manifest, pkg, host)
+ print('%s %s\n %s\n %s' % (pkg, version, info['url'], info['hash']))
+ if not info['available']:
+ print('%s marked unavailable for %s' % (pkg, host))
+ raise AssertionError
+ fetch(info['url'])
+ return info
+
def fetch_std(manifest, targets):
- stds = []
- for target in targets:
- info = fetch_package(manifest, 'rust-std', target)
- stds.append(info)
- return stds
+ stds = []
+ for target in targets:
+ info = fetch_package(manifest, 'rust-std', target)
+ stds.append(info)
+ return stds
+
def tar_for_host(host):
- if 'linux' in host:
- tar_options = 'cJf'
- tar_ext = '.tar.xz'
- else:
- tar_options = 'cjf'
- tar_ext = '.tar.bz2'
- return tar_options, tar_ext
+ if 'linux' in host:
+ tar_options = 'cJf'
+ tar_ext = '.tar.xz'
+ else:
+ tar_options = 'cjf'
+ tar_ext = '.tar.bz2'
+ return tar_options, tar_ext
+
def repack(host, targets, channel='stable', suffix=''):
- print("Repacking rust for %s..." % host)
- url = 'https://static.rust-lang.org/dist/channel-rust-' + channel + '.toml'
- req = requests.get(url)
- req.raise_for_status()
- manifest = toml.loads(req.content)
- if manifest['manifest-version'] != '2':
- print('ERROR: unrecognized manifest version %s.' % manifest['manifest-version'])
- return
- print('Using manifest for rust %s as of %s.' % (channel, manifest['date']))
- print('Fetching packages...')
- rustc = fetch_package(manifest, 'rustc', host)
- cargo = fetch_package(manifest, 'cargo', host)
- stds = fetch_std(manifest, targets)
- print('Installing packages...')
- tar_basename = 'rustc-' + host
- if suffix:
- tar_basename += '-' + suffix
- tar_basename += '-repack'
- install_dir = 'rustc'
- subprocess.check_call(['rm', '-rf', install_dir])
- install(os.path.basename(rustc['url']), install_dir)
- install(os.path.basename(cargo['url']), install_dir)
- for std in stds:
- install(os.path.basename(std['url']), install_dir)
- pass
- print('Tarring %s...' % tar_basename)
- tar_options, tar_ext = tar_for_host(host)
- subprocess.check_call(['tar', tar_options, tar_basename + tar_ext, install_dir])
- subprocess.check_call(['rm', '-rf', install_dir])
+ print("Repacking rust for %s..." % host)
+ url = 'https://static.rust-lang.org/dist/channel-rust-' + channel + '.toml'
+ req = requests.get(url)
+ req.raise_for_status()
+ manifest = toml.loads(req.content)
+ if manifest['manifest-version'] != '2':
+ print('ERROR: unrecognized manifest version %s.' % manifest[
+ 'manifest-version'])
+ return
+ print('Using manifest for rust %s as of %s.' % (channel, manifest['date']))
+ print('Fetching packages...')
+ rustc = fetch_package(manifest, 'rustc', host)
+ cargo = fetch_package(manifest, 'cargo', host)
+ stds = fetch_std(manifest, targets)
+ print('Installing packages...')
+ tar_basename = 'rustc-' + host
+ if suffix:
+ tar_basename += '-' + suffix
+ tar_basename += '-repack'
+ install_dir = 'rustc'
+ subprocess.check_call(['rm', '-rf', install_dir])
+ install(os.path.basename(rustc['url']), install_dir)
+ install(os.path.basename(cargo['url']), install_dir)
+ for std in stds:
+ install(os.path.basename(std['url']), install_dir)
+ pass
+ print('Tarring %s...' % tar_basename)
+ tar_options, tar_ext = tar_for_host(host)
+ subprocess.check_call(
+ ['tar', tar_options, tar_basename + tar_ext, install_dir])
+ subprocess.check_call(['rm', '-rf', install_dir])
+
def repack_cargo(host, channel='nightly'):
- print("Repacking cargo for %s..." % host)
- # Cargo doesn't seem to have a .toml manifest.
- base_url = 'https://static.rust-lang.org/cargo-dist/'
- req = requests.get(os.path.join(base_url, 'channel-cargo-' + channel))
- req.raise_for_status()
- file = ''
- for line in req.iter_lines():
- if line.find(host) != -1:
- file = line.strip()
- if not file:
- print('No manifest entry for %s!' % host)
- return
- manifest = {
- 'date': req.headers['Last-Modified'],
- 'pkg': {
- 'cargo': {
- 'version': channel,
- 'target': {
- host: {
- 'url': os.path.join(base_url, file),
- 'hash': None,
- 'available': True,
- },
- },
- },
- },
- }
- print('Using manifest for cargo %s.' % channel)
- print('Fetching packages...')
- cargo = fetch_package(manifest, 'cargo', host)
- print('Installing packages...')
- install_dir = 'cargo'
- subprocess.check_call(['rm', '-rf', install_dir])
- install(os.path.basename(cargo['url']), install_dir)
- tar_basename = 'cargo-%s-repack' % host
- print('Tarring %s...' % tar_basename)
- tar_options, tar_ext = tar_for_host(host)
- subprocess.check_call(['tar', tar_options, tar_basename + tar_ext, install_dir])
- subprocess.check_call(['rm', '-rf', install_dir])
+ print("Repacking cargo for %s..." % host)
+ # Cargo doesn't seem to have a .toml manifest.
+ base_url = 'https://static.rust-lang.org/cargo-dist/'
+ req = requests.get(os.path.join(base_url, 'channel-cargo-' + channel))
+ req.raise_for_status()
+ file = ''
+ for line in req.iter_lines():
+ if line.find(host) != -1:
+ file = line.strip()
+ if not file:
+ print('No manifest entry for %s!' % host)
+ return
+ manifest = {
+ 'date': req.headers['Last-Modified'],
+ 'pkg': {
+ 'cargo': {
+ 'version': channel,
+ 'target': {
+ host: {
+ 'url': os.path.join(base_url, file),
+ 'hash': None,
+ 'available': True,
+ },
+ },
+ },
+ },
+ }
+ print('Using manifest for cargo %s.' % channel)
+ print('Fetching packages...')
+ cargo = fetch_package(manifest, 'cargo', host)
+ print('Installing packages...')
+ install_dir = 'cargo'
+ subprocess.check_call(['rm', '-rf', install_dir])
+ install(os.path.basename(cargo['url']), install_dir)
+ tar_basename = 'cargo-%s-repack' % host
+ print('Tarring %s...' % tar_basename)
+ tar_options, tar_ext = tar_for_host(host)
+ subprocess.check_call(
+ ['tar', tar_options, tar_basename + tar_ext, install_dir])
+ subprocess.check_call(['rm', '-rf', install_dir])
+
# rust platform triples
-android="armv7-linux-androideabi"
-android_x86="i686-linux-android"
-linux64="x86_64-unknown-linux-gnu"
-linux32="i686-unknown-linux-gnu"
-mac64="x86_64-apple-darwin"
-mac32="i686-apple-darwin"
-win64="x86_64-pc-windows-msvc"
-win32="i686-pc-windows-msvc"
+android = "armv7-linux-androideabi"
+android_x86 = "i686-linux-android"
+linux64 = "x86_64-unknown-linux-gnu"
+linux32 = "i686-unknown-linux-gnu"
+mac64 = "x86_64-apple-darwin"
+mac32 = "i686-apple-darwin"
+win64 = "x86_64-pc-windows-msvc"
+win32 = "i686-pc-windows-msvc"
if __name__ == '__main__':
- repack(mac64, [mac64, mac32])
- repack(win32, [win32])
- repack(win64, [win64])
- repack(linux64, [linux64, linux32])
- repack(linux64, [linux64, mac64, mac32], suffix='mac-cross')
- repack(linux64, [linux64, android, android_x86], suffix='android-cross')
+ repack(mac64, [mac64, mac32])
+ repack(win32, [win32])
+ repack(win64, [win64])
+ repack(linux64, [linux64, linux32])
+ repack(linux64, [linux64, mac64, mac32], suffix='mac-cross')
+ repack(linux64, [linux64, android, android_x86], suffix='android-cross')
rename from testing/docker/rust-build/splat_rust.py
rename to taskcluster/docker/rust-build/splat_rust.py
--- a/testing/docker/rust-build/splat_rust.py
+++ b/taskcluster/docker/rust-build/splat_rust.py
@@ -5,43 +5,47 @@ tree to update them to a new set of rust
'''
import json
import os.path
import sys
from collections import OrderedDict
+
def load_manifest(path):
with open(path) as f:
return json.load(f, object_pairs_hook=OrderedDict)
return None
+
def save_manifest(manifest, path):
with open(path, 'w') as f:
json.dump(manifest, f,
- indent=0, separators=(',', ': '))
+ indent=0, separators=(',', ': '))
f.write('\n')
+
def replace(manifest, stanza):
key = 'rustc'
version = stanza.get('version')
for s in manifest:
if key in s.get('filename'):
if version:
print('Replacing %s\n with %s' % (s['version'], version))
s['version'] = version
print(' old %s' % s['digest'][:12])
s['digest'] = stanza['digest']
s['size'] = stanza['size']
print(' new %s' % s['digest'][:12])
return True
- print('Warning: Could not find matching %s filename' %key)
+ print('Warning: Could not find matching %s filename' % key)
return False
+
def update_manifest(source_manifest, target, target_filename):
for stanza in source_manifest:
filename = stanza.get('filename')
if target in filename:
size = int(stanza.get('size'))
print('Found %s %d bytes' % (filename, size))
version = stanza.get('version')
if version:
@@ -94,9 +98,9 @@ if __name__ == '__main__':
sys.exit(1)
base_path = sys.argv[1]
updates = load_manifest('manifest.tt')
for target, filenames in TARGETS.items():
for target_filename in filenames:
update_manifest(updates, target,
- os.path.join(base_path, target_filename))
+ os.path.join(base_path, target_filename))
rename from testing/docker/rust-build/task.json
rename to taskcluster/docker/rust-build/task.json
rename from testing/docker/rust-build/tcbuild.py
rename to taskcluster/docker/rust-build/tcbuild.py
--- a/testing/docker/rust-build/tcbuild.py
+++ b/taskcluster/docker/rust-build/tcbuild.py
@@ -15,60 +15,70 @@ import json
import os
import shutil
import sys
import taskcluster
import tempfile
import time
import tooltool
+requests.packages.urllib3.disable_warnings()
+
+
def local_file(filename):
'''
Return a path to a file next to this script.
'''
return os.path.join(os.path.dirname(__file__), filename)
+
def read_tc_auth(tc_auth_file):
'''
Read taskcluster credentials from tc_auth_file and return them as a dict.
'''
return json.load(open(tc_auth_file, 'rb'))
+
def fill_template_dict(d, keys):
for key, val in d.items():
if isinstance(val, basestring) and '{' in val:
d[key] = val.format(**keys)
elif isinstance(val, dict):
fill_template_dict(val, keys)
+
def fill_template(template_file, keys):
'''
Take the file object template_file, parse it as JSON, and
interpolate (using str.template) its keys using keys.
'''
template = json.load(template_file)
fill_template_dict(template, keys)
return template
+
def spawn_task(queue, args):
'''
Spawn a Taskcluster task in queue using args.
'''
task_id = taskcluster.utils.slugId()
with open(local_file('task.json'), 'rb') as template:
keys = vars(args)
now = datetime.datetime.utcnow()
keys['task_created'] = now.isoformat() + 'Z'
- keys['task_deadline'] = (now + datetime.timedelta(hours=2)).isoformat() + 'Z'
- keys['artifacts_expires'] = (now + datetime.timedelta(days=1)).isoformat() + 'Z'
+ keys['task_deadline'] = (now + datetime.timedelta(
+ hours=2)).isoformat() + 'Z'
+ keys['artifacts_expires'] = (now + datetime.timedelta(
+ days=1)).isoformat() + 'Z'
payload = fill_template(template, keys)
queue.createTask(task_id, payload)
print('--- %s task %s submitted ---' % (now, task_id))
return task_id
+
def wait_for_task(queue, task_id, initial_wait=5):
'''
Wait until queue reports that task task_id is completed, and return
its run id.
Sleep for initial_wait seconds before checking status the first time.
Then poll periodically and print a running log of the task status.
'''
@@ -76,29 +86,30 @@ def wait_for_task(queue, task_id, initia
previous_state = None
have_ticks = False
while True:
res = queue.status(task_id)
state = res['status']['state']
if state != previous_state:
now = datetime.datetime.utcnow()
if have_ticks:
- sys.stdout.write('\n')
- have_ticks = False
+ sys.stdout.write('\n')
+ have_ticks = False
print('--- %s task %s %s ---' % (now, task_id, state))
previous_state = state
if state == 'completed':
return len(res['status']['runs']) - 1
if state in ('failed', 'exception'):
raise Exception('Task failed')
sys.stdout.write('.')
sys.stdout.flush()
have_ticks = True
time.sleep(10)
+
def fetch_artifact(queue, task_id, run_id, name, dest_dir):
'''
Fetch the artifact with name from task_id and run_id in queue,
write it to a file in dest_dir, and return the path to the written
file.
'''
url = queue.buildUrl('getArtifact', task_id, run_id, name)
fn = os.path.join(dest_dir, os.path.basename(name))
@@ -109,21 +120,23 @@ def fetch_artifact(queue, task_id, run_i
with open(fn, 'wb') as f:
for chunk in r.iter_content(1024):
f.write(chunk)
except requests.exceptions.HTTPError:
print('HTTP Error %d fetching %s' % (r.status_code, name))
return None
return fn
+
def make_artifact_dir(task_id, run_id):
prefix = 'tc-artifacts.%s.%d.' % (task_id, run_id)
print('making artifact dir %s' % prefix)
return tempfile.mkdtemp(prefix=prefix)
+
def fetch_artifacts(queue, task_id, run_id):
'''
Fetch all artifacts from task_id and run_id in queue, write them to
temporary files, and yield the path to each.
'''
try:
tempdir = make_artifact_dir(task_id, run_id)
res = queue.listArtifacts(task_id, run_id)
@@ -132,20 +145,21 @@ def fetch_artifacts(queue, task_id, run_
if a['name'].startswith('public/logs'):
continue
# Skip interfaces
if a['name'].startswith('private/docker-worker'):
continue
yield fetch_artifact(queue, task_id, run_id, a['name'], tempdir)
finally:
if os.path.isdir(tempdir):
- #shutil.rmtree(tempdir)
+ # shutil.rmtree(tempdir)
print('Artifacts downloaded to %s' % tempdir)
pass
+
def upload_to_tooltool(tooltool_auth, task_id, artifact):
'''
Upload artifact to tooltool using tooltool_auth as the authentication token.
Return the path to the generated tooltool manifest.
'''
try:
oldcwd = os.getcwd()
os.chdir(os.path.dirname(artifact))
@@ -163,43 +177,54 @@ def upload_to_tooltool(tooltool_auth, ta
'-m', manifest,
'--authentication-file', tooltool_auth,
'--message', 'Built from taskcluster task {}'.format(task_id),
])
return manifest
finally:
os.chdir(oldcwd)
+
def update_manifest(artifact, manifest, local_gecko_clone):
platform = 'linux'
manifest_dir = os.path.join(local_gecko_clone,
'testing', 'config', 'tooltool-manifests')
platform_dir = [p for p in os.listdir(manifest_dir)
if p.startswith(platform)][0]
tree_manifest = os.path.join(manifest_dir, platform_dir, 'releng.manifest')
print('%s -> %s' % (manifest, tree_manifest))
shutil.copyfile(manifest, tree_manifest)
+
def main():
parser = argparse.ArgumentParser(description='Build and upload binaries')
- parser.add_argument('taskcluster_auth', help='Path to a file containing Taskcluster client ID and authentication token as a JSON file in the form {"clientId": "...", "accessToken": "..."}')
- parser.add_argument('--tooltool-auth', help='Path to a file containing a tooltool authentication token valid for uploading files')
- parser.add_argument('--local-gecko-clone', help='Path to a local Gecko clone whose tooltool manifests will be updated with the newly-built binaries')
+ parser.add_argument('taskcluster_auth',
+ help='Path to a file containing Taskcluster client '
+ 'ID and authentication token as a JSON file in '
+ 'the form {"clientId": "...", "accessToken": "..."}')
+ parser.add_argument('--tooltool-auth',
+ help='Path to a file containing a tooltool '
+ 'authentication token valid for uploading files')
+ parser.add_argument('--local-gecko-clone',
+ help='Path to a local Gecko clone whose tooltool '
+ 'manifests will be updated with the newly-built binaries')
parser.add_argument('--rust-branch', default='stable',
help='Revision of the rust repository to use')
parser.add_argument('--task', help='Use an existing task')
args = parser.parse_args()
tc_auth = read_tc_auth(args.taskcluster_auth)
queue = taskcluster.Queue({'credentials': tc_auth})
if args.task:
task_id, initial_wait = args.task, 0
else:
task_id, initial_wait = spawn_task(queue, args), 25
run_id = wait_for_task(queue, task_id, initial_wait)
for artifact in fetch_artifacts(queue, task_id, run_id):
if args.tooltool_auth:
- manifest = upload_to_tooltool(args.tooltool_auth, task_id, artifact)
+ manifest = upload_to_tooltool(args.tooltool_auth, task_id,
+ artifact)
if args.local_gecko_clone:
update_manifest(artifact, manifest, args.local_gecko_clone)
+
if __name__ == '__main__':
main()
rename from testing/docker/rust-build/upload_rust.sh
rename to taskcluster/docker/rust-build/upload_rust.sh
rename from testing/docker/tester/Dockerfile
rename to taskcluster/docker/tester/Dockerfile
rename from testing/docker/tester/REGISTRY
rename to taskcluster/docker/tester/REGISTRY
rename from testing/docker/tester/VERSION
rename to taskcluster/docker/tester/VERSION
rename from testing/docker/tester/bin/test.sh
rename to taskcluster/docker/tester/bin/test.sh
rename from testing/docker/tester/dot-config/pip/pip.conf
rename to taskcluster/docker/tester/dot-config/pip/pip.conf
rename from testing/docker/tester/dot-config/user-dirs.dirs
rename to taskcluster/docker/tester/dot-config/user-dirs.dirs
rename from testing/docker/tester/dot-config/user-dirs.locale
rename to taskcluster/docker/tester/dot-config/user-dirs.locale
rename from testing/docker/tester/dot-pulse/default.pa
rename to taskcluster/docker/tester/dot-pulse/default.pa
rename from testing/docker/tester/tc-vcs-config.yml
rename to taskcluster/docker/tester/tc-vcs-config.yml
rename from testing/docker/tester/tester.env
rename to taskcluster/docker/tester/tester.env
rename from testing/docker/upload-symbols/Dockerfile
rename to taskcluster/docker/upload-symbols/Dockerfile
rename from testing/docker/upload-symbols/README.md
rename to taskcluster/docker/upload-symbols/README.md
rename from testing/docker/upload-symbols/bin/checkout-script.sh
rename to taskcluster/docker/upload-symbols/bin/checkout-script.sh
rename from testing/docker/upload-symbols/bin/upload.sh
rename to taskcluster/docker/upload-symbols/bin/upload.sh
rename from testing/docker/upload-symbols/test_exports.sh
rename to taskcluster/docker/upload-symbols/test_exports.sh
--- a/taskcluster/docs/docker-images.rst
+++ b/taskcluster/docs/docker-images.rst
@@ -1,16 +1,16 @@
.. taskcluster_dockerimages:
=============
Docker Images
=============
TaskCluster Docker images are defined in the source directory under
-``testing/docker``. Each directory therein contains the name of an
+``taskcluster/docker``. Each directory therein contains the name of an
image used as part of the task graph.
Adding Extra Files to Images
============================
Dockerfile syntax has been extended to allow *any* file from the
source checkout to be added to the image build *context*. (Traditionally
you can only ``ADD`` files from the same directory as the Dockerfile.)
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -133,17 +133,17 @@ docker-image
------------
Tasks of the ``docker-image`` kind build the Docker images in which other
Docker tasks run.
The tasks to generate each docker image have predictable labels:
``build-docker-image-<name>``.
-Docker images are built from subdirectories of ``testing/docker``, using
+Docker images are built from subdirectories of ``taskcluster/docker``, using
``docker build``. There is currently no capability for one Docker image to
depend on another in-tree docker image, without uploading the latter to a
Docker repository
The task definition used to create the image-building tasks is given in
``image.yml`` in the kind directory, and is interpreted as a :doc:`YAML
Template <yaml-templates>`.
--- a/taskcluster/scripts/builder/build-haz-linux.sh
+++ b/taskcluster/scripts/builder/build-haz-linux.sh
@@ -49,17 +49,17 @@ GECKO_DIR=$( cd "$GECKO_DIR" && pwd )
# Directory to populate with tooltool-installed tools
export TOOLTOOL_DIR="$WORKSPACE"
# Directory to hold the (useless) object files generated by the analysis.
export MOZ_OBJDIR="$WORKSPACE/obj-analyzed"
mkdir -p "$MOZ_OBJDIR"
if [ -n "$DO_TOOLTOOL" ]; then
- ( cd $TOOLTOOL_DIR; python $GECKO_DIR/testing/docker/recipes/tooltool.py --url https://api.pub.build.mozilla.org/tooltool/ -m $GECKO_DIR/$TOOLTOOL_MANIFEST fetch -c $TOOLTOOL_CACHE )
+ ( cd $TOOLTOOL_DIR; python $GECKO_DIR/taskcluster/docker/recipes/tooltool.py --url https://api.pub.build.mozilla.org/tooltool/ -m $GECKO_DIR/$TOOLTOOL_MANIFEST fetch -c $TOOLTOOL_CACHE )
fi
export NO_MERCURIAL_SETUP_CHECK=1
if [[ "$PROJECT" = "browser" ]]; then (
cd "$WORKSPACE"
set "$WORKSPACE"
. setup-ccache.sh
--- a/taskcluster/scripts/builder/sm-tooltool-config.sh
+++ b/taskcluster/scripts/builder/sm-tooltool-config.sh
@@ -42,9 +42,9 @@ esac
# necessary for the JS shell, but it's less duplication to share tooltool
# manifests.
BROWSER_PLATFORM=$PLATFORM_OS$BITS
: ${TOOLTOOL_MANIFEST:=browser/config/tooltool-manifests/$BROWSER_PLATFORM/releng.manifest}
: ${TOOLTOOL_CHECKOUT:=$WORK}
export TOOLTOOL_CHECKOUT
-(cd $TOOLTOOL_CHECKOUT && python ${SRCDIR}/testing/docker/recipes/tooltool.py --url $TOOLTOOL_SERVER -m $SRCDIR/$TOOLTOOL_MANIFEST fetch ${TOOLTOOL_CACHE:+ -c $TOOLTOOL_CACHE})
+(cd $TOOLTOOL_CHECKOUT && python ${SRCDIR}/taskcluster/docker/recipes/tooltool.py --url $TOOLTOOL_SERVER -m $SRCDIR/$TOOLTOOL_MANIFEST fetch ${TOOLTOOL_CACHE:+ -c $TOOLTOOL_CACHE})
--- a/taskcluster/taskgraph/docker.py
+++ b/taskcluster/taskgraph/docker.py
@@ -15,23 +15,23 @@ import tempfile
import urllib2
import which
from subprocess import Popen, PIPE
from io import BytesIO
from taskgraph.util import docker
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
-IMAGE_DIR = os.path.join(GECKO, 'testing', 'docker')
+IMAGE_DIR = os.path.join(GECKO, 'taskcluster', 'docker')
INDEX_URL = 'https://index.taskcluster.net/v1/task/' + docker.INDEX_PREFIX + '.{}.{}.hash.{}'
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
def load_image_by_name(image_name, tag=None):
- context_path = os.path.join(GECKO, 'testing', 'docker', image_name)
+ context_path = os.path.join(GECKO, 'taskcluster', 'docker', image_name)
context_hash = docker.generate_context_hash(GECKO, context_path, image_name)
image_index_url = INDEX_URL.format('level-3', image_name, context_hash)
print("Fetching", image_index_url)
task = json.load(urllib2.urlopen(image_index_url))
return load_image_by_task_id(task['taskId'], tag)
--- a/taskcluster/taskgraph/task/docker_image.py
+++ b/taskcluster/taskgraph/task/docker_image.py
@@ -54,17 +54,17 @@ class DockerImageTask(base.Task):
.format(repo=params['head_repository'], rev=params['head_rev']),
'index_image_prefix': INDEX_PREFIX,
'artifact_path': 'public/image.tar.zst',
}
tasks = []
templates = Templates(path)
for image_name, image_symbol in config['images'].iteritems():
- context_path = os.path.join('testing', 'docker', image_name)
+ context_path = os.path.join('taskcluster', 'docker', image_name)
context_hash = generate_context_hash(GECKO, context_path, image_name)
image_parameters = dict(parameters)
image_parameters['image_name'] = image_name
image_parameters['context_hash'] = context_hash
image_task = templates.load('image.yml', image_parameters)
attributes = {'image_name': image_name}
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -136,17 +136,17 @@ task_description_schema = Schema({
# For tasks that will run in docker-worker or docker-engine, this is the
# name of the docker image or in-tree docker image to run the task in. If
# in-tree, then a dependency will be created automatically. This is
# generally `desktop-test`, or an image that acts an awful lot like it.
Required('docker-image'): Any(
# a raw Docker image path (repo/image:tag)
basestring,
- # an in-tree generated docker image (from `testing/docker/<name>`)
+ # an in-tree generated docker image (from `taskcluster/docker/<name>`)
{'in-tree': basestring}
),
# worker features that should be enabled
Required('relengapi-proxy', default=False): bool,
Required('chain-of-trust', default=False): bool,
Required('taskcluster-proxy', default=False): bool,
Required('allow-ptrace', default=False): bool,
--- a/taskcluster/taskgraph/transforms/tests/test_description.py
+++ b/taskcluster/taskgraph/transforms/tests/test_description.py
@@ -121,17 +121,17 @@ test_description_schema = Schema({
# For tasks that will run in docker-worker or docker-engine, this is the
# name of the docker image or in-tree docker image to run the task in. If
# in-tree, then a dependency will be created automatically. This is
# generally `desktop-test`, or an image that acts an awful lot like it.
Required('docker-image', default={'in-tree': 'desktop-test'}): Any(
# a raw Docker image path (repo/image:tag)
basestring,
- # an in-tree generated docker image (from `testing/docker/<name>`)
+ # an in-tree generated docker image (from `taskcluster/docker/<name>`)
{'in-tree': basestring}
),
# seconds of runtime after which the task will be killed. Like 'chunks',
# this can be keyed by test pltaform.
Required('max-run-time', default=3600): Any(
int,
{'by-test-platform': {basestring: int}},
--- a/taskcluster/taskgraph/util/docker.py
+++ b/taskcluster/taskgraph/util/docker.py
@@ -12,17 +12,17 @@ import tarfile
import tempfile
from mozpack.archive import (
create_tar_gz_from_files,
)
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
-DOCKER_ROOT = os.path.join(GECKO, 'testing', 'docker')
+DOCKER_ROOT = os.path.join(GECKO, 'taskcluster', 'docker')
INDEX_PREFIX = 'docker.images.v2'
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
def docker_image(name, default_version=None):
'''Determine the docker image name, including repository and tag, from an
in-tree docker file.'''
try: