docker - ubuntu18.04 streamlining final revision draft
authorEdwin Takahashi <egao@mozilla.com>
Mon, 25 Nov 2019 11:36:43 -0800
changeset 2496020 0e16a4351573a4e7fa58aa0331833545b91a30ae
parent 2495539 b4755981c1382cb88fed4e4fcff3ba73779b2080
child 2496021 f74b9acf84773d7edbb5ddd3bc30d4bfa3f71b29
child 2496364 22e4735a4e04ccea4174d6a93d6d88ab8aed6938
child 2496467 dd18403213454c4c321d8540eb45b2b28819357d
child 2496699 f5d6d6975221434a786cdd405f67a2f8b1c1ae9c
push id455266
push useregao@mozilla.com
push dateMon, 25 Nov 2019 19:38:28 +0000
treeherdertry@f74b9acf8477 [default view] [failures only]
milestone72.0a1
docker - ubuntu18.04 streamlining final revision
taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh
taskcluster/docker/ubuntu1804-test/Dockerfile
taskcluster/docker/ubuntu1804-test/autostart/jockey-gtk.desktop
taskcluster/docker/ubuntu1804-test/release-upgrades
--- a/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh
+++ b/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh
@@ -2,19 +2,24 @@
 
 set -ve
 
 test "$(whoami)" == 'root'
 
 mkdir -p /setup
 cd /setup
 
-# enable i386 packages
+# Tell APT to not install recommended packages
+echo 'APT::Install-Recommends "false";' > /etc/apt/apt.conf
+
+# Enable i386 packages
 dpkg --add-architecture i386
 
+# To speed up docker image build times as well as number of network/disk I/O
+# build a list of packages to be installed nad call it in one go.
 apt_packages=()
 
 apt_packages+=('autoconf2.13')
 apt_packages+=('bluez-cups')
 apt_packages+=('build-essential')
 apt_packages+=('ca-certificates')
 apt_packages+=('ccache')
 apt_packages+=('curl')
@@ -28,16 +33,17 @@ apt_packages+=('fonts-vlgothic')
 apt_packages+=('g++-multilib')
 apt_packages+=('gcc-multilib')
 apt_packages+=('gir1.2-gnomebluetooth-1.0')
 apt_packages+=('git')
 apt_packages+=('gstreamer1.0-plugins-base')
 apt_packages+=('gstreamer1.0-plugins-good')
 apt_packages+=('gstreamer1.0-tools')
 apt_packages+=('language-pack-en-base')
+apt_packages+=('libc6-dbg')
 apt_packages+=('libasound2-dev')
 apt_packages+=('libcanberra-gtk3-module')
 apt_packages+=('libcanberra-pulse')
 apt_packages+=('libcurl4-openssl-dev')
 apt_packages+=('libdbus-1-dev')
 apt_packages+=('libdbus-glib-1-dev')
 apt_packages+=('libfreetype6')
 apt_packages+=('libgconf2-dev')
@@ -48,32 +54,29 @@ apt_packages+=('libgtk-3-0')
 apt_packages+=('libiw-dev')
 apt_packages+=('libnotify-dev')
 apt_packages+=('libpulse-dev')
 apt_packages+=('libxt-dev')
 apt_packages+=('libxxf86vm1')
 apt_packages+=('llvm')
 apt_packages+=('llvm-dev')
 apt_packages+=('llvm-runtime')
-apt_packages+=('nano')
 apt_packages+=('net-tools')
 apt_packages+=('pulseaudio')
 apt_packages+=('pulseaudio-module-bluetooth')
 apt_packages+=('pulseaudio-module-gconf')
 apt_packages+=('qemu-kvm')
 apt_packages+=('rlwrap')
 apt_packages+=('screen')
 apt_packages+=('software-properties-common')
 apt_packages+=('sudo')
-apt_packages+=('tar')
 apt_packages+=('ttf-dejavu')
 apt_packages+=('ubuntu-desktop')
 apt_packages+=('unzip')
 apt_packages+=('uuid')
-apt_packages+=('vim')
 apt_packages+=('wget')
 apt_packages+=('xvfb')
 apt_packages+=('yasm')
 apt_packages+=('zip')
 
 # Make sure we have X libraries for 32-bit tests
 apt_packages+=('libxt6:i386')
 apt_packages+=('libpulse0:i386')
@@ -93,32 +96,34 @@ apt_packages+=('x11vnc')
 
 # Bug 1176031: need `xset` to disable screensavers
 apt_packages+=('x11-xserver-utils')
 
 # use Ubuntu's Python-2.7 (2.7.3 on Precise)
 apt_packages+=('python-dev')
 apt_packages+=('python-pip')
 
+# APT update takes unexpectedly long on Ubuntu.
+# Run it at the last possible minute.
 apt-get update
+
 # This allows ubuntu-desktop to be installed without human interaction
 export DEBIAN_FRONTEND=noninteractive
-apt-get install -y -f "${apt_packages[@]}"
+apt-get install -y -f "${apt_packages[@]}" && rm -rf /var/lib/apt/lists/*
 
-dpkg-reconfigure locales
-
+# Install tooltool, mercurial and node now that dependencies are in place.
 . /setup/common.sh
 . /setup/install-mercurial.sh
+. /setup/install-node.sh
 
+# Install pip and virutalenv
 pip install --upgrade pip==19.2.3
 hash -r
 pip install virtualenv==15.2.0
 
-. /setup/install-node.sh
-
 # Install custom-built Debian packages.  These come from a set of repositories
 # packaged in tarballs on tooltool to make them replicable.  Because they have
 # inter-dependenices, we install all repositories first, then perform the
 # installation.
 cp /etc/apt/sources.list sources.list.orig
 
 # Install Valgrind (trunk, late Jan 2016) and do some crude sanity
 # checks.  It has to go in /usr/local, otherwise it won't work.  Copy
@@ -140,46 +145,62 @@ EOF
 cp valgrind-15775-3206-ubuntu1204.tgz /tmp
 (cd / && tar xzf /tmp/valgrind-15775-3206-ubuntu1204.tgz)
 rm /tmp/valgrind-15775-3206-ubuntu1204.tgz
 cp /usr/local/bin/valgrind /usr/bin/valgrind
 apt-get install -y libc6-dbg
 valgrind --version
 valgrind date
 
-# Until bug 1511527 is fixed, remove the file from the image to ensure it's not there.
-# rm -f /usr/local/bin/linux64-minidump_stackwalk
-
 # adding multiverse to get 'ubuntu-restricted-extras' below
 apt-add-repository multiverse
 apt-get update
 
 # for mp4 codec (used in MSE tests)
-apt-get -q -y -f install ubuntu-restricted-extras
-# TEMPORARY: we do not want flash installed, but the above pulls it in (bug 1349208)
-rm -f /usr/lib/flashplugin-installer/libflashplayer.so
+apt-get -q -y -f install ubuntu-restricted-extras && rm -rf /var/lib/apt/lists/*
 
 apt-get -q -y -f install \
     libxcb1 \
     libxcb-render0 \
     libxcb-shm0 \
     libxcb-glx0 \
-    libxcb-shape0
+    libxcb-shape0 &&
+    rm -rf /var/lib/apt/lists/*
 
 apt-get -q -y -f install \
     libgl1-mesa-dri \
     libgl1-mesa-glx \
-    mesa-common-dev
+    mesa-common-dev &&
+    rm -rf /var/lib/apt/lists/*
 
 # revert the list of repos
 cp sources.list.orig /etc/apt/sources.list
 
-# clean up
-apt-get -y autoremove
+# Build a list of packages to purge from the image.
+apt_packages=()
+apt_packages+=('git')
+apt_packages+=('ubuntu-release-upgrader-core')
+apt_packages+=('ubuntu-release-upgrader-gtk')
+apt_packages+=('update-manager-core')
+apt_packages+=('update-manager')
+apt_packages+=('yelp')
+
+# Purge unnecessary packages
+apt-get purge -y -f "${apt_packages[@]}"
+
+# Clear apt cache one last time
+rm -rf /var/cache/apt/archives
 
 # We don't need no docs!
 rm -rf /usr/share/help /usr/share/doc /usr/share/man
 
+# Remove all locale files other than en_US.UTF-8
+rm -rf /usr/share/locale/   /usr/share/locale-langpack/     /usr/share/locales/
+echo "en_US.UTF-8 UTF-8" > /var/lib/locales/supported.d/en
+locale-gen
+
+# Further cleanup
 cd /
 rm -rf /setup ~/.ccache ~/.cache ~/.npm
+apt-get -y autoremove
 apt-get clean
 apt-get autoclean
 rm -f "$0"
--- a/taskcluster/docker/ubuntu1804-test/Dockerfile
+++ b/taskcluster/docker/ubuntu1804-test/Dockerfile
@@ -1,26 +1,31 @@
 # The base image is a last known working snapshot of this image. We do this
 # to work around problems rebuilding desktop1604-test. See bug 1503756 for
 # the ugly details. Bug 1511527 tracks undoing this hack and building from
 # a fresh ubuntu image.
 FROM          ubuntu:18.04
 MAINTAINER    Edwin Takahashi <egao@mozilla.com>
 
-RUN mkdir -p /builds
-RUN id worker || useradd -d /builds/worker -s /bin/bash -m worker
+# Create necessary directories and worker user account
+RUN     mkdir -p /builds && mkdir -p artifacts
+RUN     id worker || useradd -d /builds/worker -s /bin/bash -m worker
 WORKDIR /builds/worker
 
 # We need to declare all potentially cache volumes as caches. Also,
 # making high I/O paths volumes increase I/O throughput because of
 # AUFS slowness.
-VOLUME /builds/worker/.cache
-VOLUME /builds/worker/checkouts
-VOLUME /builds/worker/tooltool-cache
-VOLUME /builds/worker/workspace
+VOLUME  /builds/worker/.cache
+VOLUME  /builds/worker/checkouts
+VOLUME  /builds/worker/tooltool-cache
+VOLUME  /builds/worker/workspace
+
+# In test.sh we accept START_VNC to start a vnc daemon.
+# Exposing this port allows it to work.
+EXPOSE  5900
 
 # %include python/mozbuild/mozbuild/action/tooltool.py
 ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py
 
 # %include testing/mozharness/external_tools/robustcheckout.py
 ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
 
 # %include taskcluster/docker/recipes/hgrc
@@ -30,92 +35,80 @@ COPY topsrcdir/taskcluster/docker/recipe
 ADD topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh
 
 # %include taskcluster/docker/recipes/install-mercurial.sh
 ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh
 
 # %include taskcluster/docker/recipes/install-node.sh
 ADD topsrcdir/taskcluster/docker/recipes/install-node.sh /setup/install-node.sh
 
-# %include taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh
-ADD topsrcdir/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh /setup/system-setup.sh
-RUN           bash /setup/system-setup.sh
-
 # Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb
 # %include taskcluster/docker/recipes/xvfb.sh
 ADD topsrcdir/taskcluster/docker/recipes/xvfb.sh /builds/worker/scripts/xvfb.sh
 
 # %include taskcluster/scripts/run-task
 ADD topsrcdir/taskcluster/scripts/run-task /builds/worker/bin/run-task
 
 # %include taskcluster/scripts/misc/fetch-content
 ADD topsrcdir/taskcluster/scripts/misc/fetch-content /builds/worker/bin/fetch-content
 
+ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /builds/worker/scripts/tooltool.py
+
+# %include taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh
+ADD topsrcdir/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh /setup/system-setup.sh
+RUN     bash /setup/system-setup.sh
+
 # %include taskcluster/scripts/tester/test-linux.sh
 ADD topsrcdir/taskcluster/scripts/tester/test-linux.sh /builds/worker/bin/test-linux.sh
 
-ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /builds/worker/scripts/tooltool.py
-
 # Set variable normally configured at login, by the shells parent process, these
 # are taken from GNU su manual
-ENV           HOME          /builds/worker
-ENV           SHELL         /bin/bash
-ENV           USER          worker
-ENV           LOGNAME       worker
-ENV           HOSTNAME      taskcluster-worker
-ENV           LANG          en_US.UTF-8
-ENV           LC_ALL        en_US.UTF-8
-
-# Add utilities and configuration
-COPY         dot-files/config              /builds/worker/.config
-COPY         dot-files/pulse               /builds/worker/.pulse
-RUN          chmod +x bin/*
-
-# allow the worker user to access video devices
-RUN usermod -a -G video worker
-
-RUN mkdir -p artifacts
-
-ENV PATH $PATH:/builds/worker/bin
-
-# Disable Ubuntu update prompt
-# http://askubuntu.com/questions/515161/ubuntu-12-04-disable-release-notification-of-14-04-in-update-manager
-ADD release-upgrades /etc/update-manager/release-upgrades
-
-# Disable tools with on-login popups that interfere with tests; see bug 1240084 and bug 984944.
-ADD autostart/jockey-gtk.desktop autostart/deja-dup-monitor.desktop /etc/xdg/autostart/
-
-# Bug 1345105 - Do not run periodical update checks and downloads
-ADD autostart/gnome-software-service.desktop /etc/xdg/autostart/
-
-# In test.sh we accept START_VNC to start a vnc daemon.
-# Exposing this port allows it to work.
-EXPOSE 5900
+ENV     HOME            /builds/worker
+ENV     SHELL           /bin/bash
+ENV     USER            worker
+ENV     LOGNAME         worker
+ENV     HOSTNAME        taskcluster-worker
+ENV     LANG            en_US.UTF-8
+ENV     LC_ALL          en_US.UTF-8
+ENV     PATH            $PATH:/builds/worker/bin
 
 # This helps not forgetting setting DISPLAY=:0 when running
 # tests outside of test.sh
 ENV DISPLAY :0
 
+# Add utilities and configuration
+COPY    dot-files/config        /builds/worker/.config
+COPY    dot-files/pulse         /builds/worker/.pulse
+
 # Disable apport (app crash reporter) to avoid stealing focus from test runs
 ADD apport /etc/default/apport
 
 # Disable font antialiasing for now to match releng's setup
 ADD fonts.conf /builds/worker/.fonts.conf
 
+# Bug 1345105 - Do not run periodical update checks and downloads
+ADD autostart/gnome-software-service.desktop /etc/xdg/autostart/
+
+# allow the worker user to access video devices
+RUN     usermod -a -G video worker
+
+# Set execution and ownership privileges
+RUN     chmod +x bin/*; chown -R worker:worker /builds/worker
+
 # Set up first-run experience for interactive mode
 ADD motd /etc/taskcluster-motd
 ADD taskcluster-interactive-shell /bin/taskcluster-interactive-shell
-RUN chmod +x /bin/taskcluster-interactive-shell
-
-RUN chown -R worker:worker /builds/worker
+RUN     chmod +x /bin/taskcluster-interactive-shell
 
 # gnome-keyring-daemon is configured to have the IPC_LOCK capability (to lock pages with secrets in
 # memory), but docker isn't run with that capability granted. So, if we were to try running
 # gnome-keyring-daemon without first clearing the capability, it would just exit with the message
 # "Operation not permitted". Luckily it doesn't actually require the capability.
-RUN setcap -r /usr/bin/gnome-keyring-daemon
+RUN     setcap -r /usr/bin/gnome-keyring-daemon
 
+# Ubuntu 18.04 (or any other GNOME3 based systems) needs dbus to have various
+# test function as expected. Use entrypoint to initialize dbus as root.
 COPY dbus.sh /usr/local/bin/dbus.sh
 RUN chmod +x /usr/local/bin/dbus.sh
 ENTRYPOINT ["/usr/local/bin/dbus.sh"]
 
 # Set a default command useful for debugging
 CMD ["/bin/bash", "--login"]
deleted file mode 100644
--- a/taskcluster/docker/ubuntu1804-test/autostart/jockey-gtk.desktop
+++ /dev/null
@@ -1,15 +0,0 @@
-[Desktop Entry]
-Name=Check for new hardware drivers
-Comment=Notify about new hardware drivers available for the system
-Icon=jockey
-Exec=sh -c "test -e /var/cache/jockey/check || exec jockey-gtk --check"
-Terminal=false
-Type=Application
-Categories=System;Settings;GTK;HardwareSettings;
-NotShowIn=KDE;
-X-Ubuntu-Gettext-Domain=jockey
-
-# Bug 984944/1240084 - It prevents taking screenshots
-X-GNOME-Autostart-Delay=false
-
-NoDisplay=true
deleted file mode 100644
--- a/taskcluster/docker/ubuntu1804-test/release-upgrades
+++ /dev/null
@@ -1,17 +0,0 @@
-# Default behavior for the release upgrader.
-
-[DEFAULT]
-# Default prompting behavior, valid options:
-#
-#  never  - Never check for a new release.
-#  normal - Check to see if a new release is available.  If more than one new
-#           release is found, the release upgrader will attempt to upgrade to
-#           the release that immediately succeeds the currently-running
-#           release.
-#  lts    - Check to see if a new LTS release is available.  The upgrader
-#           will attempt to upgrade to the first LTS release available after
-#           the currently-running one.  Note that this option should not be
-#           used if the currently-running release is not itself an LTS
-#           release, since in that case the upgrader won't be able to
-#           determine if a newer release is available.
-Prompt=never