Bug 1490573 - Use fetches for nsis and wine toolchain tasks. r=froydnj
authorMike Hommey <mh+mozilla@glandium.org>
Mon, 11 Mar 2019 01:38:46 +0000
changeset 521467 8ecd607b3d513cf2646705ccb6c29fc0a463d959
parent 521466 e7f9e5754e7741b0fb521b797deb68802738046f
child 521468 29b1d70e8032850d0649e2c6f791c9e4e8d23620
push id10866
push usernerli@mozilla.com
push dateTue, 12 Mar 2019 18:59:09 +0000
treeherdermozilla-beta@445c24a51727 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfroydnj
bugs1490573
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1490573 - Use fetches for nsis and wine toolchain tasks. r=froydnj These toolchain tasks are the last ones using the historical download-tools script from build/unix/build-gcc, which invokes gpg to validate the downloaded tarballs. The consequence is that gpg-agent is spawned and stays running, preventing a cleanup script from doing its job, making the tasks fail. Fetches are the new way to download sources, and can also do gpg validation without those caveats. The download-tools.sh script can then be removed as it's not used anymore. Differential Revision: https://phabricator.services.mozilla.com/D22682
build/unix/build-gcc/build-gcc.sh
build/unix/build-gcc/download-tools.sh
taskcluster/ci/fetch/toolchains.yml
taskcluster/ci/toolchain/mingw.yml
taskcluster/scripts/misc/build-mingw32-nsis.sh
taskcluster/scripts/misc/build-wine.sh
--- a/build/unix/build-gcc/build-gcc.sh
+++ b/build/unix/build-gcc/build-gcc.sh
@@ -1,17 +1,15 @@
 #!/bin/bash
 
 set -e
 set -x
 
 make_flags="-j$(nproc)"
 
-. $data_dir/download-tools.sh
-
 prepare_mingw() {
   export prefix=/tools/mingw32
   export install_dir=$root_dir$prefix
   mkdir -p $install_dir
   export PATH=$PATH:$install_dir/bin/
 
   cd $root_dir
 
deleted file mode 100755
--- a/build/unix/build-gcc/download-tools.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/bin/bash
-
-set -e
-set -x
-
-if [ -z "$root_dir" -o ! -d "$root_dir" ]; then
-  root_dir=$(mktemp -d)
-fi
-
-if test -z $TMPDIR; then
-  TMPDIR=/tmp/
-fi
-
-mkdir $root_dir/gpg
-GPG="gpg --homedir $root_dir/gpg"
-
-> $root_dir/downloads
-
-download() {
-  wget -c --progress=dot:mega -P $TMPDIR $1/$2
-  (cd $TMPDIR; sha256sum $2) >> $root_dir/downloads
-}
-
-download_and_check() {
-  download $1 ${2%.*}
-  wget -c --progress=dot:mega -P $TMPDIR $1/$2
-  $GPG --verify $TMPDIR/$2 $TMPDIR/${2%.*}
-}
--- a/taskcluster/ci/fetch/toolchains.yml
+++ b/taskcluster/ci/fetch/toolchains.yml
@@ -201,8 +201,38 @@ wix-3.1.1:
 
 grcov-osx-x86_64:
   description: grcov binary release
   fetch:
     type: static-url
     url: https://github.com/mozilla/grcov/releases/download/v0.3.2/grcov-osx-x86_64.tar.bz2
     sha256: 6a810f782efb207234ef418ab136edd1c0ffa61aa84d8c4545042a6fa3b9dd13
     size: 915948
+
+zlib-1.2.11:
+  description: zlib 1.2.11 source code
+  fetch:
+    type: static-url
+    url: http://zlib.net/zlib-1.2.11.tar.gz
+    sha256: c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
+    size: 607698
+    gpg-signature:
+      sig-url: "{url}.asc"
+      key-path: build/unix/build-gcc/5ED46A6721D365587791E2AA783FCD8E58BCAFBA.key
+
+nsis-3.01:
+  description: nsis 3.01 source code
+  fetch:
+    type: static-url
+    url: https://downloads.sourceforge.net/project/nsis/NSIS%203/3.01/nsis-3.01-src.tar.bz2
+    sha256: 604c011593be484e65b2141c50a018f1b28ab28c994268e4ecd377773f3ffba1
+    size: 1649428
+
+wine-3.0.3:
+  description: wine-3.0.3.tar.xz
+  fetch:
+    type: static-url
+    url: http://dl.winehq.org/wine/source/3.0/wine-3.0.3.tar.xz
+    sha256: eb645999ea6f6455a5275bf267e19a32497c8f5aac818ea40afe7c8c396a4da1
+    size: 19735412
+    gpg-signature:
+      sig-url: "{url}.sign"
+      key-path: build/unix/build-gcc/DA23579A74D4AD9AF9D3F945CEFAC8EAAF17519D.key
--- a/taskcluster/ci/toolchain/mingw.yml
+++ b/taskcluster/ci/toolchain/mingw.yml
@@ -22,16 +22,19 @@ linux64-wine:
     description: "Wine build for MinGW32 Cross Compile"
     treeherder:
         symbol: TMW(wine)
     worker:
         max-run-time: 10800
     run:
         script: build-wine.sh
         toolchain-artifact: public/build/wine.tar.xz
+    fetches:
+        fetch:
+            - wine-3.0.3
 
 linux64-mingw32-gcc:
     description: "GCC toolchain build for MinGW32 Cross Compile"
     treeherder:
         symbol: TMW(mingw32-gcc)
     worker:
         max-run-time: 10800
     run:
@@ -55,16 +58,20 @@ linux64-mingw32-nsis:
     run:
         script: build-mingw32-nsis.sh
         resources:
             - 'build/unix/build-gcc/build-gcc.sh'
             - 'taskcluster/scripts/misc/build-gcc-mingw32.sh'
         toolchain-artifact: public/build/nsis.tar.xz
     toolchains:
         - linux64-mingw32-gcc
+    fetches:
+        fetch:
+            - nsis-3.01
+            - zlib-1.2.11
 
 linux64-mingw-fxc2-x86:
     description: "fxc2.exe x86 build for MinGW Cross Compile"
     treeherder:
         symbol: TMW(mingw-fxc2-x86)
     worker:
         max-run-time: 1800
     run:
--- a/taskcluster/scripts/misc/build-mingw32-nsis.sh
+++ b/taskcluster/scripts/misc/build-mingw32-nsis.sh
@@ -14,50 +14,36 @@ set -x -e -v
 WORKSPACE=$HOME/workspace
 HOME_DIR=$WORKSPACE/build
 INSTALL_DIR=$WORKSPACE/build/src/mingw32
 TOOLTOOL_DIR=$WORKSPACE/build/src
 UPLOAD_DIR=$HOME/artifacts
 
 mkdir -p $INSTALL_DIR
 
-root_dir=$HOME_DIR
-data_dir=$HOME_DIR/src/build/unix/build-gcc
-
-. $data_dir/download-tools.sh
-
 cd $TOOLTOOL_DIR
 . taskcluster/scripts/misc/tooltool-download.sh
 # After tooltool runs, we move the stuff we just downloaded.
 # As explained above, we have to build nsis to the directory it
 # will eventually be run from, which is the same place we just
 # installed our compiler. But at the end of the script we want
 # to package up what we just built. If we don't move the compiler,
 # we will package up the compiler we downloaded along with the
 # stuff we just built.
 mv mingw32 mingw32-gcc
 export PATH="$TOOLTOOL_DIR/mingw32-gcc/bin:$PATH"
 
-cd $WORKSPACE
-
-$GPG --import $data_dir/5ED46A6721D365587791E2AA783FCD8E58BCAFBA.key
+cd $HOME_DIR
 
 # --------------
 
-download_and_check http://zlib.net/ zlib-1.2.11.tar.gz.asc
-tar xaf $TMPDIR/zlib-1.2.11.tar.gz
 cd zlib-1.2.11
 make -f win32/Makefile.gcc PREFIX=i686-w64-mingw32-
 
-cd ..
-wget --progress=dot:mega https://downloads.sourceforge.net/project/nsis/NSIS%203/3.01/nsis-3.01-src.tar.bz2
-echo "559703cc25f78697be1784a38d1d9a19c97d27a200dc9257d1483c028c6be9242cbcd10391ba618f88561c2ba57fdbd8b3607bea47ed8c3ad7509a6ae4075138  nsis-3.01-src.tar.bz2" | sha512sum -c -
-bunzip2 nsis-3.01-src.tar.bz2
-tar xaf nsis-3.01-src.tar
-cd nsis-3.01-src
+cd ../nsis-3.01-src
 # I don't know how to make the version work with the environment variables/config flags the way the author appears to
 sed -i "s/'VERSION', 'Version of NSIS', cvs_version/'VERSION', 'Version of NSIS', '3.01'/" SConstruct
 scons XGCC_W32_PREFIX=i686-w64-mingw32- ZLIB_W32=../zlib-1.2.11 SKIPUTILS="NSIS Menu" PREFIX=$INSTALL_DIR/ install
 
 # --------------
 
 cd $WORKSPACE/build/src
 tar caf nsis.tar.xz mingw32
--- a/taskcluster/scripts/misc/build-wine.sh
+++ b/taskcluster/scripts/misc/build-wine.sh
@@ -3,27 +3,19 @@ set -x -e -v
 
 WORKSPACE=$HOME/workspace
 HOME_DIR=$WORKSPACE/build
 INSTALL_DIR=$WORKSPACE/wine
 UPLOAD_DIR=$HOME/artifacts
 
 mkdir -p $INSTALL_DIR
 
-root_dir=$HOME_DIR
-data_dir=$HOME_DIR/src/build/unix/build-gcc
-
-. $data_dir/download-tools.sh
-
-cd $WORKSPACE
+cd $HOME_DIR
 
 # --------------
-$GPG --import $data_dir/DA23579A74D4AD9AF9D3F945CEFAC8EAAF17519D.key
-download_and_check http://dl.winehq.org/wine/source/3.0/ wine-3.0.3.tar.xz.sign
-tar xaf $TMPDIR/wine-3.0.3.tar.xz
 cd wine-3.0.3
 ./configure --prefix=$INSTALL_DIR/
 make -j$(nproc)
 make install
 
 # --------------
 
 cd $WORKSPACE/