Bug 1534533 - Add geckodriver toolchain tasks. r=chmanchester
authorNick Alexander <nalexander@mozilla.com>
Wed, 28 Aug 2019 20:54:06 +0000
changeset 490473 d2bdeab3b280bd01bbcef4279ba2d05895e82cfd
parent 490472 ded4116115e840093904991347a4aca03c9b0a9b
child 490474 c88dc69728eded1efdeca1676a4a1ceb1473d9c7
push id36504
push userccoroiu@mozilla.com
push dateThu, 29 Aug 2019 04:08:39 +0000
treeherdermozilla-central@7004b8779a36 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschmanchester
bugs1534533
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1534533 - Add geckodriver toolchain tasks. r=chmanchester There was quite a bit of discussion of this in `#build` on IRC, and the consensus was that geckodriver should be built as a stand-alone Rust crate and not as part of Firefox/Gecko (say, as a new --enable-project target). This follows that approach, and the expression, modeled off of cbindgen but updated to cross compile from a Linux host to all targets, is pretty straight-forward. A sparse profile would be nice, but the way that the Gecko Cargo workspace works means that the profile must accumulate Rust code from many locations. If we want to, eventually testing/geckodriver can be removed from the top-level Rust workspace, the geckodriver-signing tasks migrated to these toolchain tasks, consumers migrated to the signing tasks, and geckodriver removed from the "common" test archive. Differential Revision: https://phabricator.services.mozilla.com/D43646
taskcluster/ci/toolchain/geckodriver.yml
taskcluster/ci/toolchain/kind.yml
taskcluster/ci/toolchain/rust.yml
taskcluster/scripts/misc/build-cbindgen.sh
taskcluster/scripts/misc/build-geckodriver.sh
taskcluster/scripts/misc/build-sccache.sh
taskcluster/scripts/misc/osx-cross-linker
taskcluster/scripts/misc/vs-setup.sh
taskcluster/scripts/misc/vs-setup32.sh
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/toolchain/geckodriver.yml
@@ -0,0 +1,80 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+job-defaults:
+    description: "geckodriver toolchain build"
+    worker-type: b-linux
+    worker:
+        max-run-time: 1800
+    run:
+        script: build-geckodriver.sh
+        toolchain-artifact: public/build/geckodriver.tar.xz
+        sparse-profile: null
+        resources:
+            - 'testing/geckodriver'
+            - 'testing/mozbase/rust'
+            - 'testing/webdriver'
+            # Changes to 'third_party/rust' should be captured by
+            # 'testing/geckodriver/Cargo.toml'.
+
+linux64-geckodriver:
+    treeherder:
+        symbol: TL(geckodriver)
+    run:
+        arguments: ['x86_64-unknown-linux-gnu']
+    fetches:
+        toolchain:
+            - linux64-rust
+
+macosx64-geckodriver:
+    treeherder:
+        symbol: TM(geckodriver)
+    worker:
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
+    run:
+        arguments: ['x86_64-apple-darwin']
+        resources:
+            - taskcluster/scripts/misc/tooltool-download.sh
+        tooltool-downloads: internal
+    fetches:
+        toolchain:
+            - linux64-cctools-port
+            - linux64-clang
+            - linux64-llvm-dsymutil
+            - linux64-rust-macos
+
+win32-geckodriver:
+    treeherder:
+        symbol: TW32(geckodriver)
+    worker:
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
+    run:
+        arguments: ['i686-pc-windows-msvc']
+        resources:
+            - taskcluster/scripts/misc/tooltool-download.sh
+        toolchain-artifact: public/build/geckodriver.tar.bz2
+        tooltool-downloads: internal
+    fetches:
+        toolchain:
+            - linux64-clang
+            - linux64-rust-windows
+
+win64-geckodriver:
+    treeherder:
+        symbol: TW64(geckodriver)
+    worker:
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/vs2017.manifest"
+    run:
+        arguments: ['x86_64-pc-windows-msvc']
+        resources:
+            - taskcluster/scripts/misc/tooltool-download.sh
+        toolchain-artifact: public/build/geckodriver.tar.bz2
+        tooltool-downloads: internal
+    fetches:
+        toolchain:
+            - linux64-clang
+            - linux64-rust-windows
--- a/taskcluster/ci/toolchain/kind.yml
+++ b/taskcluster/ci/toolchain/kind.yml
@@ -24,16 +24,17 @@ job-defaults:
 
 jobs-from:
     - android.yml
     - cbindgen.yml
     - cctools-port.yml
     - clang-tidy.yml
     - clang.yml
     - gcc.yml
+    - geckodriver.yml
     - gn.yml
     - grcov.yml
     - mingw.yml
     - minidump_stackwalk.yml
     - misc.yml
     - nasm.yml
     - node.yml
     - rust.yml
--- a/taskcluster/ci/toolchain/rust.yml
+++ b/taskcluster/ci/toolchain/rust.yml
@@ -218,16 +218,30 @@ linux64-rust-android-1.37:
             '--target', 'armv7-linux-androideabi',
             '--target', 'thumbv7neon-linux-androideabi',
             '--target', 'aarch64-linux-android',
             '--target', 'i686-linux-android',
             '--target', 'x86_64-linux-android',
         ]
         toolchain-alias: linux64-rust-android
 
+linux64-rust-windows-1.37:
+    description: "rust repack with windows-cross support"
+    treeherder:
+        symbol: TL(rust-win)
+    run:
+        arguments: [
+            '--channel', '1.37.0',
+            '--host', 'x86_64-unknown-linux-gnu',
+            '--target', 'x86_64-unknown-linux-gnu',
+            '--target', 'x86_64-pc-windows-msvc',
+            '--target', 'i686-pc-windows-msvc',
+        ]
+        toolchain-alias: linux64-rust-windows
+
 win64-rust-1.28:
     treeherder:
         symbol: TW64(rust-1.28)
     run:
         arguments: [
             '--channel', '1.28.0',
             '--host', 'x86_64-pc-windows-msvc',
             '--target', 'x86_64-pc-windows-msvc',
--- a/taskcluster/scripts/misc/build-cbindgen.sh
+++ b/taskcluster/scripts/misc/build-cbindgen.sh
@@ -1,12 +1,13 @@
 #!/bin/bash
 set -x -e -v
 
-TARGET="$1"
+# Needed by osx-cross-linker.
+export TARGET="$1"
 
 case "$(uname -s)" in
 Linux)
     COMPRESS_EXT=xz
     ;;
 MINGW*)
     UPLOAD_DIR=$PWD/public/build
     COMPRESS_EXT=bz2
@@ -22,27 +23,17 @@ cd $GECKO_PATH
 if [ -n "$TOOLTOOL_MANIFEST" ]; then
   . taskcluster/scripts/misc/tooltool-download.sh
 fi
 
 # OSX cross builds are a bit harder
 if [ "$TARGET" == "x86_64-apple-darwin" ]; then
   export PATH="$MOZ_FETCHES_DIR/llvm-dsymutil/bin:$PATH"
   export PATH="$MOZ_FETCHES_DIR/cctools/bin:$PATH"
-  cat >cross-linker <<EOF
-exec $MOZ_FETCHES_DIR/clang/bin/clang -v \
-  -fuse-ld=$MOZ_FETCHES_DIR/cctools/bin/x86_64-apple-darwin-ld \
-  -mmacosx-version-min=10.11 \
-  -target $TARGET \
-  -B $MOZ_FETCHES_DIR/cctools/bin \
-  -isysroot $MOZ_FETCHES_DIR/MacOSX10.11.sdk \
-  "\$@"
-EOF
-  chmod +x cross-linker
-  export RUSTFLAGS="-C linker=$PWD/cross-linker"
+  export RUSTFLAGS="-C linker=$GECKO_PATH/taskcluster/scripts/misc/osx-cross-linker"
 fi
 
 export PATH="$(cd $MOZ_FETCHES_DIR && pwd)/rustc/bin:$PATH"
 
 cd $MOZ_FETCHES_DIR/cbindgen
 
 cargo build --verbose --release --target "$TARGET"
 
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/misc/build-geckodriver.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+set -x -e -v
+
+# Needed by osx-cross-linker.
+export TARGET="$1"
+
+cd $GECKO_PATH
+
+if [ -n "$TOOLTOOL_MANIFEST" ]; then
+  . taskcluster/scripts/misc/tooltool-download.sh
+fi
+
+EXE=
+COMPRESS_EXT=xz
+
+case "$TARGET" in
+*windows-msvc)
+  EXE=.exe
+  COMPRESS_EXT=bz2
+  if [[ $TARGET == "i686-pc-windows-msvc" ]]; then
+    . $GECKO_PATH/taskcluster/scripts/misc/vs-setup32.sh
+    export CARGO_TARGET_I686_PC_WINDOWS_MSVC_LINKER=$MOZ_FETCHES_DIR/clang/bin/lld-link
+  else
+    . $GECKO_PATH/taskcluster/scripts/misc/vs-setup.sh
+    export CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER=$MOZ_FETCHES_DIR/clang/bin/lld-link
+  fi
+  ( IFS=\;
+    for d in $LIB; do
+      (cd "$d"; rename y/A-Z/a-z/ *)
+    done
+  )
+  ;;
+# OSX cross builds are a bit harder
+x86_64-apple-darwin)
+  export PATH="$MOZ_FETCHES_DIR/llvm-dsymutil/bin:$PATH"
+  export PATH="$MOZ_FETCHES_DIR/cctools/bin:$PATH"
+  export RUSTFLAGS="-C linker=$GECKO_PATH/taskcluster/scripts/misc/osx-cross-linker"
+  ;;
+esac
+
+export PATH="$(cd $MOZ_FETCHES_DIR && pwd)/rustc/bin:$PATH"
+
+cd $GECKO_PATH/testing/geckodriver
+
+cargo build --frozen --verbose --release --target "$TARGET"
+
+cd $GECKO_PATH
+
+cp target/$TARGET/release/geckodriver$EXE .
+tar -acf geckodriver.tar.$COMPRESS_EXT geckodriver$EXE
+mkdir -p $UPLOAD_DIR
+cp geckodriver.tar.$COMPRESS_EXT $UPLOAD_DIR
+
+. $GECKO_PATH/taskcluster/scripts/misc/vs-cleanup.sh
--- a/taskcluster/scripts/misc/build-sccache.sh
+++ b/taskcluster/scripts/misc/build-sccache.sh
@@ -1,12 +1,13 @@
 #!/bin/bash
 set -x -e -v
 
-TARGET="$1"
+# Needed by osx-cross-linker.
+export TARGET="$1"
 
 # This script is for building sccache
 
 case "$(uname -s)" in
 Linux)
     COMPRESS_EXT=xz
     PATH="$MOZ_FETCHES_DIR/binutils/bin:$PATH"
     ;;
@@ -28,27 +29,17 @@ PATH="$(cd $MOZ_FETCHES_DIR && pwd)/rust
 
 cd $MOZ_FETCHES_DIR/sccache
 
 case "$(uname -s)" in
 Linux)
     if [ "$TARGET" == "x86_64-apple-darwin" ]; then
         export PATH="$MOZ_FETCHES_DIR/llvm-dsymutil/bin:$PATH"
         export PATH="$MOZ_FETCHES_DIR/cctools/bin:$PATH"
-        cat >cross-linker <<EOF
-exec $MOZ_FETCHES_DIR/clang/bin/clang -v \
-  -fuse-ld=$MOZ_FETCHES_DIR/cctools/bin/x86_64-apple-darwin-ld \
-  -mmacosx-version-min=10.11 \
-  -target $TARGET \
-  -B $MOZ_FETCHES_DIR/cctools/bin \
-  -isysroot $MOZ_FETCHES_DIR/MacOSX10.11.sdk \
-  "\$@"
-EOF
-        chmod +x cross-linker
-        export RUSTFLAGS="-C linker=$PWD/cross-linker"
+        export RUSTFLAGS="-C linker=$GECKO_PATH/taskcluster/scripts/misc/osx-cross-linker"
         export CC="$MOZ_FETCHES_DIR/clang/bin/clang"
         cargo build --features "all" --verbose --release --target $TARGET
     else
         # We can't use the system openssl; see the sad story in
         # https://bugzilla.mozilla.org/show_bug.cgi?id=1163171#c26.
         OPENSSL_BUILD_DIRECTORY=$PWD/ourssl
         pushd $MOZ_FETCHES_DIR/openssl-1.1.0g
         ./Configure --prefix=$OPENSSL_BUILD_DIRECTORY no-shared linux-x86_64
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/misc/osx-cross-linker
@@ -0,0 +1,7 @@
+exec $MOZ_FETCHES_DIR/clang/bin/clang -v \
+  -fuse-ld=$MOZ_FETCHES_DIR/cctools/bin/x86_64-apple-darwin-ld \
+  -mmacosx-version-min=10.11 \
+  -target $TARGET \
+  -B $MOZ_FETCHES_DIR/cctools/bin \
+  -isysroot $MOZ_FETCHES_DIR/MacOSX10.11.sdk \
+  "$@"
--- a/taskcluster/scripts/misc/vs-setup.sh
+++ b/taskcluster/scripts/misc/vs-setup.sh
@@ -1,8 +1,8 @@
 VSDIR=vs2017_15.8.4
 VSPATH="${MOZ_FETCHES_DIR}/${VSDIR}"
 UNIX_VSPATH="$(cd ${MOZ_FETCHES_DIR} && pwd)/${VSDIR}"
 SDK_VERSION=10.0.17134.0
 
 export INCLUDE="${VSPATH}/VC/include;${VSPATH}/VC/atlmfc/include;${VSPATH}/SDK/Include/${SDK_VERSION}/ucrt;${VSPATH}/SDK/Include/${SDK_VERSION}/shared;${VSPATH}/SDK/Include/${SDK_VERSION}/um;${VSPATH}/SDK/Include/${SDK_VERSION}/winrt;${VSPATH}/DIA SDK/include"
-export LIB="${VSPATH}/VC/lib/x64;${VSPATH}/VC/atlmfc/lib/x64;${VSPATH}/SDK/lib/${SDK_VERSION}/um/x64;${VSPATH}/SDK/lib/${SDK_VERSION}/ucrt/x64;${VSPATH}/DIA SDK/lib/amd64"
+export LIB="${VSPATH}/VC/lib/x64;${VSPATH}/VC/atlmfc/lib/x64;${VSPATH}/SDK/Lib/${SDK_VERSION}/um/x64;${VSPATH}/SDK/Lib/${SDK_VERSION}/ucrt/x64;${VSPATH}/DIA SDK/lib/amd64"
 export PATH="${UNIX_VSPATH}/VC/bin/Hostx64/x64:${UNIX_VSPATH}/VC/bin/Hostx86/x86:${UNIX_VSPATH}/SDK/bin/${SDK_VERSION}/x64:${UNIX_VSPATH}/redist/x64/Microsoft.VC141.CRT:${UNIX_VSPATH}/SDK/Redist/ucrt/DLLs/x64:${UNIX_VSPATH}/DIA SDK/bin/amd64:$PATH"
copy from taskcluster/scripts/misc/vs-setup.sh
copy to taskcluster/scripts/misc/vs-setup32.sh
--- a/taskcluster/scripts/misc/vs-setup.sh
+++ b/taskcluster/scripts/misc/vs-setup32.sh
@@ -1,8 +1,8 @@
 VSDIR=vs2017_15.8.4
 VSPATH="${MOZ_FETCHES_DIR}/${VSDIR}"
 UNIX_VSPATH="$(cd ${MOZ_FETCHES_DIR} && pwd)/${VSDIR}"
 SDK_VERSION=10.0.17134.0
 
 export INCLUDE="${VSPATH}/VC/include;${VSPATH}/VC/atlmfc/include;${VSPATH}/SDK/Include/${SDK_VERSION}/ucrt;${VSPATH}/SDK/Include/${SDK_VERSION}/shared;${VSPATH}/SDK/Include/${SDK_VERSION}/um;${VSPATH}/SDK/Include/${SDK_VERSION}/winrt;${VSPATH}/DIA SDK/include"
-export LIB="${VSPATH}/VC/lib/x64;${VSPATH}/VC/atlmfc/lib/x64;${VSPATH}/SDK/lib/${SDK_VERSION}/um/x64;${VSPATH}/SDK/lib/${SDK_VERSION}/ucrt/x64;${VSPATH}/DIA SDK/lib/amd64"
-export PATH="${UNIX_VSPATH}/VC/bin/Hostx64/x64:${UNIX_VSPATH}/VC/bin/Hostx86/x86:${UNIX_VSPATH}/SDK/bin/${SDK_VERSION}/x64:${UNIX_VSPATH}/redist/x64/Microsoft.VC141.CRT:${UNIX_VSPATH}/SDK/Redist/ucrt/DLLs/x64:${UNIX_VSPATH}/DIA SDK/bin/amd64:$PATH"
+export LIB="${VSPATH}/VC/lib/x86;${VSPATH}/VC/atlmfc/lib/x86;${VSPATH}/SDK/Lib/${SDK_VERSION}/um/x86;${VSPATH}/SDK/Lib/${SDK_VERSION}/ucrt/x86;${VSPATH}/DIA SDK/lib"
+export PATH="${UNIX_VSPATH}/VC/bin/Hostx64/x86:${UNIX_VSPATH}/VC/bin/Hostx64/x64:${UNIX_VSPATH}/VC/bin/Hostx86/x86:${UNIX_VSPATH}/SDK/bin/${SDK_VERSION}/x86:${UNIX_VSPATH}/VC/redist/x86/Microsoft.VC141.CRT:${UNIX_VSPATH}/SDK/Redist/ucrt/DLLs/x86:${UNIX_VSPATH}/DIA SDK/bin:$PATH"