Bug 1329282 - QEMU image for building docker images
This adds a QEMU image based on Ubuntu 16.04 for in-tree
docker images. Changes to docker-image tasks are not part
of this commit.
MozReview-Commit-ID: FnSGRfMCCEn
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/config.yml
@@ -0,0 +1,19 @@
+entrypoint: ['bash', '-bash', '-c', 'exec "$@"', '--']
+user: worker
+env:
+ # Set variable normally configured at login, by the shells parent process, these
+ # are taken from GNU su manual
+ HOME: /home/worker
+ SHELL: /bin/bash
+ USER: worker
+ LOGNAME: worker
+ HOSTNAME: taskcluster-worker
+ LC_ALL: C
+ # Set terminal emulator
+ TERM: xterm
+ # Set HG_STORE_PATH for run-task
+ HG_STORE_PATH: /home/worker/checkouts/hg-store
+ # Set MOZ_AUTOMATION to make mach and other scripts stop complaining
+ MOZ_AUTOMATION: '1'
+shell: ['bash', '-bash', '-li']
+workdir: /home/worker
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/install.sh
@@ -0,0 +1,47 @@
+#!/bin/bash -e
+
+echo ' - Removing password from "worker" user'
+passwd -d worker
+
+echo ' - Allow "worker" user to do sudo without password'
+echo 'worker ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
+
+echo ' - Installing taskcluster-worker qemu-guest-tools'
+cp /tmp/custom-data/config.yml /etc/taskcluster-worker-qemu-guest-tools.yml
+cp /tmp/custom-data/taskcluster-worker /usr/local/bin/taskcluster-worker
+chmod +x /usr/local/bin/taskcluster-worker
+cp /tmp/custom-data/taskcluster-worker.service /etc/systemd/system/taskcluster-worker.service
+chmod 644 /etc/systemd/system/taskcluster-worker.service
+systemctl enable taskcluster-worker.service
+
+echo ' - Installing docker'
+apt-get install -y apt-transport-https ca-certificates
+apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+echo 'deb https://apt.dockerproject.org/repo ubuntu-xenial main' > /etc/apt/sources.list.d/docker.list
+apt-get update -y
+apt-get install -y docker-engine
+
+echo ' - Grant worker user access to docker'
+usermod -aG docker worker
+
+echo ' - Installing python'
+apt-get install -y python
+
+echo ' - Installing mercurial'
+mkdir -p /setup /build
+cp /tmp/custom-data/tooltool.py /setup/tooltool.py
+cp /tmp/custom-data/tooltool.py /setup/tooltool.py
+mkdir -p /usr/local/mercurial
+cp /tmp/custom-data/robustcheckout.py /usr/local/mercurial/robustcheckout.py
+. /tmp/custom-data/common.sh
+. /tmp/custom-data/install-mercurial.sh
+
+echo ' - Configuring mercurial to use uncompressed bundles'
+echo -e '[ui]\nclonebundleprefers = VERSION=packed1' > /home/worker/.hgrc
+
+echo ' - Installing test script'
+cp /tmp/custom-data/run-task /usr/local/bin/run-task;
+chmod +x /usr/local/bin/run-task;
+
+echo ' - chown ~/'
+chown -R worker:worker /home/worker/
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/taskcluster-worker.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=QEMU guest tools for taskcluster-worker
+After=docker.service
+Requires=network-online.target
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/taskcluster-worker qemu-guest-tools -c /etc/taskcluster-worker-qemu-guest-tools.yml
+
+[Install]
+WantedBy=multi-user.target
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/worker.seed
@@ -0,0 +1,101 @@
+# Defaults from: https://github.com/netson/ubuntu-unattended/blob/master/netson.seed
+# More good docs at: https://www.debian.org/releases/jessie/amd64/apbs04.html.en
+
+# Always install the virtual kernel
+d-i base-installer/kernel/override-image string linux-virtual
+
+# Localization
+d-i debian-installer/language string en_US:en
+d-i debian-installer/country string US
+d-i debian-installer/locale string en_US
+d-i localechooser/supported-locales multiselect en_US.UTF-8
+d-i pkgsel/install-language-support boolean false
+
+# Keyboard selection
+d-i console-setup/ask_detect boolean false
+d-i keyboard-configuration/modelcode string pc105
+d-i keyboard-configuration/layoutcode string us
+d-i keyboard-configuration/variantcode string intl
+d-i keyboard-configuration/xkb-keymap select us(intl)
+d-i debconf/language string en_US:en
+
+# Network settings
+d-i netcfg/choose_interface select auto
+d-i netcfg/dhcp_timeout string 30
+d-i netcfg/get_hostname string worker-vm
+d-i netcfg/get_domain string worker-vm
+d-i hw-detect/load_firmware boolean true
+
+# Mirror settings
+d-i mirror/country string manual
+d-i mirror/http/hostname string archive.ubuntu.com
+d-i mirror/http/directory string /ubuntu
+d-i mirror/http/proxy string
+
+# Configure apt
+d-i apt-setup/restricted boolean true
+d-i apt-setup/universe boolean true
+d-i apt-setup/backports boolean true
+d-i apt-setup/services-select multiselect security
+d-i apt-setup/security_host string security.ubuntu.com
+d-i apt-setup/security_path string /ubuntu
+tasksel tasksel/first multiselect Basic Ubuntu server
+d-i pkgsel/upgrade select full-upgrade
+d-i pkgsel/update-policy select none
+d-i pkgsel/updatedb boolean true
+d-i debconf debconf/frontend select Noninteractive
+popularity-contest popularity-contest/participate boolean false
+
+# User configuration...
+d-i passwd/root-login boolean false
+d-i passwd/make-user boolean true
+d-i passwd/user-fullname string worker
+d-i passwd/username string worker
+d-i passwd/user-password password worker
+d-i passwd/user-password-again password worker
+d-i passwd/user-uid string
+d-i user-setup/allow-password-weak boolean true
+d-i user-setup/encrypt-home boolean false
+
+# Clock and time (use UTC, don't sync with NTP, timezone UTC)
+d-i clock-setup/utc boolean true
+d-i clock-setup/ntp boolean false
+d-i time/zone string UTC
+
+# Partitioning
+d-i partman/confirm_write_new_label boolean true
+d-i partman/choose_partition select finish
+d-i partman/confirm_nooverwrite boolean true
+d-i partman/confirm boolean true
+d-i partman-auto/purge_lvm_from_device boolean true
+d-i partman-lvm/device_remove_lvm boolean true
+d-i partman-lvm/confirm boolean true
+d-i partman-lvm/confirm_nooverwrite boolean true
+d-i partman-auto-lvm/no_boot boolean true
+d-i partman-md/device_remove_md boolean true
+d-i partman-md/confirm boolean true
+d-i partman-md/confirm_nooverwrite boolean true
+d-i partman-auto/method string lvm
+d-i partman-auto-lvm/guided_size string max
+d-i partman-partitioning/confirm_write_new_label boolean true
+
+# Package selection
+d-i pkgsel/include string curl
+
+# No verbose output and no boot splash screen.
+d-i debian-installer/quiet boolean true
+d-i debian-installer/splash boolean false
+
+# Bootloader
+d-i grub-installer/timeout string 0
+d-i grub-installer/only_debian boolean true
+d-i grub-installer/with_other_os boolean true
+
+# Command to run before finishing install
+d-i preseed/late_command string /cdrom/custom-data/wrap-install.sh
+
+# Poweroff the machine after install
+d-i finish-install/reboot_in_progress note
+d-i finish-install/keep-consoles boolean false
+d-i cdrom-detect/eject boolean false
+d-i debian-installer/exit/poweroff boolean true
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/custom-data/wrap-install.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+cp -r /cdrom/custom-data /target/tmp/custom-data
+chmod +x /target/tmp/custom-data/install.sh
+chroot /target /bin/bash --login /tmp/custom-data/install.sh | /cdrom/custom-data/taskcluster-worker qemu-guest-tools post-log -
+rm -rf /target/tmp/custom-data
new file mode 100644
--- /dev/null
+++ b/taskcluster/qemu/docker-image-builder/image.yml
@@ -0,0 +1,82 @@
+symbol: 'I(docker-image-builder)'
+private: false
+description: |
+ Automatically built Ubuntu image for building in-tree docker images.
+ This image contains
+ * python 2.7
+ * mercurial
+ * docker
+ * run-task (from taskcluster/docker/recipes/run-task)
+disksize: 30 # GB
+machine: # virtual machine definition
+ uuid: 52bab607-10f1-4049-a0f8-ee4725cb715b
+ chipset: pc-i440fx-2.8
+ usb: nec-usb-xhci
+ network: e1000
+ mac: aa:54:1a:30:5c:de
+ storage: virtio-blk-pci
+ graphics: qxl-vga
+ sound: none
+ keyboard: usb-kbd
+ keyboardLayout: en-us
+ mouse: usb-mouse
+ tablet: usb-tablet
+cdromA:
+ # Download an extract ubuntu 16.04 server install media
+ - extract:
+ url: http://releases.ubuntu.com/16.04.2/ubuntu-16.04.2-server-amd64.iso
+ sha256: 737ae7041212c628de5751d15c3016058b0e833fdc32e7420209b76ca3d0a535
+ format: iso
+ target: /
+
+ # Modify the grub boot menu to have a timeout of 1 second
+ - sed: 's/timeout\s\+[0-9]\+/timeout 1/g'
+ target: /isolinux/isolinux.cfg
+
+ # Modify boot options to specify preseed in kernel parameters
+ - sed: '/label install/ilabel autoinstall\nmenu label ^Autoinstall Ubuntu Worker\nkernel /install/vmlinuz\nappend file=/cdrom/custom-data/worker.seed initrd=/install/initrd.gz auto=true priority=high preseed/file=/cdrom/custom-data/worker.seed --'
+ target: /isolinux/txt.cfg
+
+ # Add custom-data which contains preseed, install scripts, etc
+ - copy: ./custom-data
+ target: /custom-data
+ - chmod: +x
+ target: /custom-data/wrap-install.sh
+
+ # Add run-task and install-mercurial.sh as needed by custom-data/install.sh
+ - copy: /taskcluster/docker/recipes/run-task
+ target: /custom-data/run-task
+ - copy: /python/mozbuild/mozbuild/action/tooltool.py
+ target: /custom-data/tooltool.py
+ - copy: /taskcluster/docker/recipes/common.sh
+ target: /custom-data/common.sh
+ - copy: /testing/mozharness/external_tools/robustcheckout.py
+ target: /custom-data/robustcheckout.py
+ - copy: /taskcluster/docker/recipes/install-mercurial.sh
+ target: /custom-data/install-mercurial.sh
+
+ # Download taskcluster-worker and add it to custom-data
+ - copy:
+ url: https://github.com/taskcluster/taskcluster-worker/releases/download/v0.1.9/taskcluster-worker-0.1.9-linux-amd64
+ sha256: dc184f3c741ed4098c05350c531739f176d6e2146d38314311e6b019b5f84727
+ target: /custom-data/taskcluster-worker
+
+ # Make taskcluster-worker binary executable
+ - chmod: +x
+ target: /custom-data/taskcluster-worker
+
+ # Package everything into an ISO
+ - genisoimage:
+ - '-JlDrV'
+ - 'UBUNTU_INSTALLER'
+ - '-input-charset'
+ - 'utf-8'
+ - '-cache-inodes'
+ - '-b'
+ - 'isolinux/isolinux.bin'
+ - '-c'
+ - 'isolinux/boot.cat'
+ - '-no-emul-boot'
+ - '-boot-load-size'
+ - '4'
+ - '-boot-info-table'