Backout changeset 8f7958c7d3e5 (bug 1431523) to give time to docker images, etc. to build without blocking other landings.
authorMike Hommey <mh+mozilla@glandium.org>
Thu, 14 Feb 2019 09:37:02 +0900
changeset 458958 6ad8d2be5c2c
parent 458957 8f7958c7d3e5
child 458959 d09c7dc93f20
push id111906
push usermh@glandium.org
push dateThu, 14 Feb 2019 00:39:30 +0000
treeherdermozilla-inbound@6ad8d2be5c2c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1431523
milestone67.0a1
backs out8f7958c7d3e5
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backout changeset 8f7958c7d3e5 (bug 1431523) to give time to docker images, etc. to build without blocking other landings.
taskcluster/ci/docker-image/kind.yml
taskcluster/docker/debian-base/Dockerfile
taskcluster/docker/debian-base/cloud-mirror-workaround.sh
taskcluster/docker/debian-base/setup_packages.sh
taskcluster/docker/debian-packages/Dockerfile
taskcluster/docker/debian-raw/Dockerfile
taskcluster/docker/debian-raw/cloud-mirror-workaround.sh
taskcluster/docker/debian-raw/setup_packages.sh
taskcluster/taskgraph/transforms/job/debian_package.py
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -17,44 +17,23 @@ transforms:
 # (to use subdirectory clones of the proper directory), at which point we can
 # generate tasks for every docker image in the directory, secure in the
 # knowledge that unnecessary images will be omitted from the target task graph
 jobs:
   image_builder:
     symbol: I(ib)
   desktop1604-test:
     symbol: I(dt16t)
-  # Neither the debian7-*raw nor the debian7-*packages images can have
-  # packages dependencies.
-  debian7-raw:
-    symbol: I(deb7-raw)
-    definition: debian-raw
-    args:
-      BASE_IMAGE: debian:wheezy-20171210
-      DIST: wheezy
-      SNAPSHOT: '20171210T214726Z'
-  debian7-packages:
-    symbol: I(deb7-pkg)
-    definition: debian-packages
-    parent: debian7-raw
-  debian7-i386-raw:
-    symbol: I(deb7-32-raw)
-    definition: debian-raw
-    args:
-      BASE_IMAGE: i386/debian:wheezy-20171210
-      DIST: wheezy
-      SNAPSHOT: '20171210T214726Z'
-  debian7-i386-packages:
-    symbol: I(deb7-32-pkg)
-    definition: debian-packages
-    parent: debian7-i386-raw
   debian7-base:
     symbol: I(deb7-base)
     definition: debian-base
-    parent: debian7-raw
+    args:
+      DIST: wheezy
+      BASE_TAG: '20171210'
+      SNAPSHOT: '20171210T214726Z'
     packages:
       - deb7-gdb
       - deb7-git
       - deb7-make
       - deb7-mercurial
       - deb7-python
       - deb7-python3.5
       - deb7-python3-defaults
@@ -127,33 +106,23 @@ jobs:
     parent: debian7-amd64-build
     packages:
       - deb7-cmake
   valgrind-build:
     symbol: I(vb)
     parent: debian7-amd64-build
   lint:
     symbol: I(lnt)
-  # Neither the debian9-raw nor the debian9-packages images can have
-  # packages dependencies.
-  debian9-raw:
-    symbol: I(deb9-raw)
-    definition: debian-raw
-    args:
-      BASE_IMAGE: debian:stretch-20170620
-      DIST: stretch
-      SNAPSHOT: '20170830T000511Z'
-  debian9-packages:
-    symbol: I(deb9-pkg)
-    definition: debian-packages
-    parent: debian9-raw
   debian9-base:
     symbol: I(deb9-base)
     definition: debian-base
-    parent: debian9-raw
+    args:
+      DIST: stretch
+      BASE_TAG: '20170620'
+      SNAPSHOT: '20170830T000511Z'
     packages:
       - deb9-mercurial
       - deb9-python-zstandard
   debian9-amd64-build:
     symbol: I(deb9)
     parent: debian9-base
     definition: debian-build
     args:
--- a/taskcluster/docker/debian-base/Dockerfile
+++ b/taskcluster/docker/debian-base/Dockerfile
@@ -1,10 +1,11 @@
-# %ARG DOCKER_IMAGE_PARENT
-FROM $DOCKER_IMAGE_PARENT
+# %ARG DIST
+# %ARG BASE_TAG
+FROM debian:$DIST-$BASE_TAG
 MAINTAINER Mike Hommey <mhommey@mozilla.com>
 
 ### Add worker user and setup its workspace.
 RUN mkdir /builds && \
     groupadd -g 1000 worker && \
     useradd -u 1000 -g 1000 -d /builds/worker -s /bin/bash -m worker && \
     mkdir -p /builds/worker/workspace && \
     chown -R worker:worker /builds
@@ -17,24 +18,46 @@ VOLUME /builds/worker/workspace
 VOLUME /builds/worker/tooltool-cache
 
 # Set variable normally configured at login, by the shells parent process, these
 # are taken from GNU su manual
 ENV HOME=/builds/worker \
     SHELL=/bin/bash \
     USER=worker \
     LOGNAME=worker \
-    HOSTNAME=taskcluster-worker
+    HOSTNAME=taskcluster-worker \
+    DEBIAN_FRONTEND=noninteractive
 
 # Set a default command useful for debugging
 CMD ["/bin/bash", "--login"]
 
+# %ARG SNAPSHOT
+# Set apt sources list to a snapshot.
+RUN for s in debian_$DIST debian_$DIST-updates debian_$DIST-backports debian-security_$DIST/updates; do \
+      echo "deb http://snapshot.debian.org/archive/${s%_*}/$SNAPSHOT/ ${s#*_} main"; \
+    done > /etc/apt/sources.list && \
+    ( echo 'quiet "true";'; \
+      echo 'APT::Get::Assume-Yes "true";'; \
+      echo 'APT::Install-Recommends "false";'; \
+      echo 'Acquire::Check-Valid-Until "false";'; \
+      echo 'Acquire::Retries "5";'; \
+    ) > /etc/apt/apt.conf.d/99taskcluster
+
+RUN apt-get update && \
+    apt-get install \
+      apt-transport-https \
+      ca-certificates
+
+COPY setup_packages.sh /usr/local/sbin/
+COPY cloud-mirror-workaround.sh /usr/local/sbin/
+
 # %ARG TASKCLUSTER_ROOT_URL
 # %ARG DOCKER_IMAGE_PACKAGES
 RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAGES && \
+    echo 'dir::bin::methods::https "/usr/local/sbin/cloud-mirror-workaround.sh";' > /etc/apt/apt.conf.d/99cloud-mirror-workaround && \
     apt-get update && \
     apt-get install \
       git \
       less \
       make \
       mercurial \
       patch \
       python \
rename from taskcluster/docker/debian-raw/cloud-mirror-workaround.sh
rename to taskcluster/docker/debian-base/cloud-mirror-workaround.sh
rename from taskcluster/docker/debian-raw/setup_packages.sh
rename to taskcluster/docker/debian-base/setup_packages.sh
deleted file mode 100644
--- a/taskcluster/docker/debian-packages/Dockerfile
+++ /dev/null
@@ -1,10 +0,0 @@
-# %ARG DOCKER_IMAGE_PARENT
-FROM $DOCKER_IMAGE_PARENT
-MAINTAINER Mike Hommey <mhommey@mozilla.com>
-
-RUN apt-get install --install-recommends \
-      apt-utils \
-      aptitude \
-      build-essential \
-      devscripts \
-      fakeroot
deleted file mode 100644
--- a/taskcluster/docker/debian-raw/Dockerfile
+++ /dev/null
@@ -1,30 +0,0 @@
-# %ARG BASE_IMAGE
-FROM $BASE_IMAGE
-MAINTAINER Mike Hommey <mhommey@mozilla.com>
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-# Set a default command useful for debugging
-CMD ["/bin/bash", "--login"]
-
-COPY setup_packages.sh /usr/local/sbin/
-COPY cloud-mirror-workaround.sh /usr/local/sbin/
-
-# %ARG DIST
-# %ARG SNAPSHOT
-# Set apt sources list to a snapshot.
-RUN for s in debian_$DIST debian_$DIST-updates debian_$DIST-backports debian-security_$DIST/updates; do \
-      echo "deb http://snapshot.debian.org/archive/${s%_*}/$SNAPSHOT/ ${s#*_} main"; \
-    done > /etc/apt/sources.list && \
-    ( echo 'quiet "true";'; \
-      echo 'APT::Get::Assume-Yes "true";'; \
-      echo 'APT::Install-Recommends "false";'; \
-      echo 'Acquire::Check-Valid-Until "false";'; \
-      echo 'Acquire::Retries "5";'; \
-      echo 'dir::bin::methods::https "/usr/local/sbin/cloud-mirror-workaround.sh";'; \
-    ) > /etc/apt/apt.conf.d/99taskcluster
-
-RUN apt-get update && \
-    apt-get install \
-      apt-transport-https \
-      ca-certificates
--- a/taskcluster/taskgraph/transforms/job/debian_package.py
+++ b/taskcluster/taskgraph/transforms/job/debian_package.py
@@ -4,16 +4,17 @@
 """
 Support for running spidermonkey jobs via dedicated scripts
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os
 import re
+import taskcluster_urls
 
 from taskgraph.util.schema import Schema
 from voluptuous import Any, Optional, Required
 
 from taskgraph.transforms.job import run_job_using
 from taskgraph.transforms.job.common import add_artifacts
 
 from taskgraph.util.hash import hash_path
@@ -67,31 +68,27 @@ run_schema = Schema({
 
 
 @run_job_using("docker-worker", "debian-package", schema=run_schema)
 def docker_worker_debian_package(config, job, taskdesc):
     run = job['run']
 
     name = taskdesc['label'].replace('{}-'.format(config.kind), '', 1)
 
+    docker_repo = 'debian'
     arch = run.get('arch', 'amd64')
+    if arch != 'amd64':
+        docker_repo = '{}/{}'.format(arch, docker_repo)
 
     worker = taskdesc['worker']
     worker['artifacts'] = []
-    version = {
-        'wheezy': 7,
-        'jessie': 8,
-        'stretch': 9,
-        'buster': 10,
-    }[run['dist']]
-    image = 'debian%d' % version
-    if arch != 'amd64':
-        image += '-' + arch
-    image += '-packages'
-    worker['docker-image'] = {'in-tree': image}
+    worker['docker-image'] = '{repo}:{dist}-{date}'.format(
+        repo=docker_repo,
+        dist=run['dist'],
+        date=run['snapshot'][:8])
     # Retry on apt-get errors.
     worker['retry-exit-status'] = [100]
 
     add_artifacts(config, job, taskdesc, path='/tmp/artifacts')
 
     env = worker.setdefault('env', {})
     env['DEBFULLNAME'] = 'Mozilla build team'
     env['DEBEMAIL'] = 'dev-builds@lists.mozilla.org'
@@ -108,30 +105,39 @@ def docker_worker_debian_package(config,
     else:
         raise RuntimeError('Unreachable')
     src_url = src['url']
     src_file = os.path.basename(src_url)
     src_sha256 = src['sha256']
     package = package_re.match(src_file).group(0)
     unpack = unpack.format(src_file=src_file, package=package)
 
+    base_deps = [
+        'apt-utils',
+        'build-essential',
+        'devscripts',
+        'fakeroot',
+    ]
+
     resolver = run.get('resolver', 'apt-get')
     if resolver == 'apt-get':
         resolver = 'apt-get -yyq --no-install-recommends'
     elif resolver == 'aptitude':
         resolver = ('aptitude -y --without-recommends -o '
                     'Aptitude::ProblemResolver::Hints::KeepBuildDeps='
                     '"reject {}-build-deps :UNINST"').format(package)
+        base_deps.append('aptitude')
     else:
         raise RuntimeError('Unreachable')
 
     adjust = ''
     if 'patch' in run:
-        # We don't use robustcheckout or run-task to get a checkout. So for
-        # this one file we'd need from a checkout, download it.
+        # We can't depend on docker images, so we don't have robustcheckout or
+        # or run-task to get a checkout. So for this one file we'd need
+        # from a checkout, download it.
         env['PATCH_URL'] = '{head_repo}/raw-file/{head_rev}/build/debian-packages/{patch}'.format(
             head_repo=config.params['head_repository'],
             head_rev=config.params['head_rev'],
             patch=run['patch'],
         )
         adjust += 'curl -sL $PATCH_URL | patch -p1 && '
     if 'pre-build-command' in run:
         adjust += run['pre-build-command'] + ' && '
@@ -143,25 +149,43 @@ def docker_worker_debian_package(config,
         )
     if 'patch' not in run and 'pre-build-command' not in run:
         adjust += ('debchange -l ".{prefix}moz" --distribution "{dist}"'
                    ' "Mozilla backport for {dist}." < /dev/null && ').format(
             prefix=name.split('-', 1)[0],
             dist=run['dist'],
         )
 
+    queue_url = taskcluster_urls.api(get_root_url(), 'queue', 'v1', '')
+
+    # We can't depend on docker images (since docker images depend on packages),
+    # so we inline the whole script here.
     worker['command'] = [
         'sh',
         '-x',
         '-c',
+        # Fill /etc/apt/sources.list with the relevant snapshot repository.
+        'echo "deb http://snapshot.debian.org/archive/debian'
+        '/{snapshot}/ {dist} main" > /etc/apt/sources.list && '
+        'echo "deb http://snapshot.debian.org/archive/debian'
+        '/{snapshot}/ {dist}-updates main" >> /etc/apt/sources.list && '
+        'echo "deb http://snapshot.debian.org/archive/debian'
+        '/{snapshot}/ {dist}-backports main" >> /etc/apt/sources.list && '
+        'echo "deb http://snapshot.debian.org/archive/debian-security'
+        '/{snapshot}/ {dist}/updates main" >> /etc/apt/sources.list && '
+        'apt-get update -o Acquire::Check-Valid-Until=false -q && '
         # Add sources for packages coming from other package tasks.
-        '/usr/local/sbin/setup_packages.sh {queue_url} $PACKAGES && '
-        'apt-get update && '
-        # Upgrade packages that might have new versions in package tasks.
-        'apt-get dist-upgrade && '
+        'apt-get install -yyq apt-transport-https ca-certificates && '
+        'for task in $PACKAGES; do '
+        '  echo "deb [trusted=yes] {queue_url}task/$task/artifacts/public/build/ debian/" '
+        '>> /etc/apt/sources.list; '
+        'done && '
+        # Install the base utilities required to build debian packages.
+        'apt-get update -o Acquire::Check-Valid-Until=false -q && '
+        'apt-get install -yyq {base_deps} && '
         'cd /tmp && '
         # Get, validate and extract the package source.
         'dget -d -u {src_url} && '
         'echo "{src_sha256}  {src_file}" | sha256sum -c && '
         '{unpack} && '
         'cd {package} && '
         # Optionally apply patch and/or pre-build command.
         '{adjust}'
@@ -172,26 +196,27 @@ def docker_worker_debian_package(config,
         # Copy the artifacts
         'mkdir -p {artifacts}/debian && '
         'dcmd cp ../{package}_*.changes {artifacts}/debian/ && '
         'cd {artifacts} && '
         # Make the artifacts directory usable as an APT repository.
         'apt-ftparchive sources debian | gzip -c9 > debian/Sources.gz && '
         'apt-ftparchive packages debian | gzip -c9 > debian/Packages.gz'
         .format(
-            queue_url=get_root_url(),
+            queue_url=queue_url,
             package=package,
             snapshot=run['snapshot'],
             dist=run['dist'],
             src_url=src_url,
             src_file=src_file,
             src_sha256=src_sha256,
             unpack=unpack,
             adjust=adjust,
             artifacts='/tmp/artifacts',
+            base_deps=' '.join(base_deps),
             resolver=resolver,
         )
     ]
 
     if run.get('packages'):
         env = worker.setdefault('env', {})
         env['PACKAGES'] = {
             'task-reference': ' '.join('<{}>'.format(p)
@@ -203,14 +228,17 @@ def docker_worker_debian_package(config,
 
     # Use the command generated above as the base for the index hash.
     # We rely on it not varying depending on the head_repository or head_rev.
     digest_data = list(worker['command'])
     if 'patch' in run:
         digest_data.append(
             hash_path(os.path.join(GECKO, 'build', 'debian-packages', run['patch'])))
 
+    if docker_repo != 'debian':
+        digest_data.append(docker_repo)
+
     if not taskgraph.fast:
         taskdesc['cache'] = {
             'type': 'packages.v1',
             'name': name,
             'digest-data': digest_data
         }