Bug 1490427 - Build and use custom d8 from a toolchain task. r=jmaher,ahal
authorGregory Mierzwinski <gmierz2@outlook.com>
Fri, 21 Dec 2018 19:21:15 +0000
changeset 508851 49dfcbef9786340590699a6090d9e34a82f6868c
parent 508850 7b61dae71e8d122292def16ea0302163138c78d0
child 508852 1357e3b67ae3e711056a31b529296236f92f3607
push id10547
push userffxbld-merge
push dateMon, 21 Jan 2019 13:03:58 +0000
treeherdermozilla-beta@24ec1916bffe [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher, ahal
bugs1490427
milestone66.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1490427 - Build and use custom d8 from a toolchain task. r=jmaher,ahal This patch adds a toolchain task for building d8 with customized build settings and uses it in jsshell benchmark tests. A customized image with a debian9-base ('custom-v8') is added by this patch as well and is required to build the tool. Differential Revision: https://phabricator.services.mozilla.com/D14019
taskcluster/ci/docker-image/kind.yml
taskcluster/ci/fetch/benchmarks.yml
taskcluster/ci/source-test/jsshell.yml
taskcluster/ci/source-test/kind.yml
taskcluster/ci/toolchain/linux.yml
taskcluster/docker/custom-v8/Dockerfile
taskcluster/scripts/misc/build-custom-v8.sh
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -41,16 +41,19 @@ jobs:
       - deb7-sqlite3
       - deb7-xz-utils
   toolchain-build:
     symbol: I(toolchain)
     parent: debian7-base
     packages:
       - deb7-cmake
       - deb7-ninja
+  custom-v8:
+    symbol: I(custom-v8)
+    parent: debian9-base
   debian7-amd64-build-base:
     symbol: I(deb7-bb)
     parent: debian7-base
     definition: debian7-build
     packages:
       - deb7-nasm
       - deb7-valgrind
     args:
--- a/taskcluster/ci/fetch/benchmarks.yml
+++ b/taskcluster/ci/fetch/benchmarks.yml
@@ -1,16 +1,8 @@
-d8:
-  description: V8 debug shell
-  fetch:
-    type: static-url
-    url: https://github.com/mozilla/perf-automation/releases/download/d8-6.7.17/d8-6.7.17.zip
-    sha256: 0aa1c4e630de78373185fc1c0fa34bc87826f63fd4cbb664668891d6f6a6b24e
-    size: 20578358
-
 octane:
   description: Octane benchmark
   fetch:
     type: static-url
     url: https://github.com/mozilla/perf-automation/releases/download/octane-v1/octane-0f5b8d48d4a9.zip
     sha256: 38425ee1abfc5feca178b2f60fbd82b5873897c345112a85359be00024402f9f
     size: 1816138
 
--- a/taskcluster/ci/source-test/jsshell.yml
+++ b/taskcluster/ci/source-test/jsshell.yml
@@ -9,34 +9,34 @@ job-defaults:
             linux64.*:
                 env:
                     by-shell:
                         sm:
                             SHELL: /bin/bash
                             JSSHELL: /home/cltbld/fetches/js
                         v8:
                             SHELL: /bin/bash
-                            JSSHELL: /home/cltbld/fetches/d8/d8
+                            JSSHELL: /home/cltbld/fetches/d8/release/d8
                 max-run-time: 1800
     treeherder:
         kind: test
         tier: 2
     run:
         using: mach
         workdir: /home/cltbld
         mach: jsshell-bench --perfherder={shell} --binary=$JSSHELL {test}
     run-on-projects:
         by-shell:
             sm: ['mozilla-central', 'try', 'integration']
             default: ['mozilla-central', 'try']
     fetches:
         build:
             - target.jsshell.zip
-        fetch:
-            - d8
+        toolchain:
+            - linux64-custom-v8
 
 bench-ares6:
     description: Ares6 JavaScript shell benchmark suite
     shell: ['sm', 'v8']
     test: ares6
     treeherder:
         symbol: ares6
 
--- a/taskcluster/ci/source-test/kind.yml
+++ b/taskcluster/ci/source-test/kind.yml
@@ -7,16 +7,17 @@ loader: taskgraph.loader.transform:loade
 transforms:
    - taskgraph.transforms.try_job:transforms
    - taskgraph.transforms.source_test:transforms
    - taskgraph.transforms.job:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
    - fetch
+   - toolchain
 
 jobs-from:
    - cram.yml
    - doc.yml
    - file-metadata.yml
    - jsshell.yml
    - mocha.yml
    - mozlint.yml
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@@ -876,16 +876,37 @@ linux64-mingw-fxc2-x86:
         resources:
             - 'build/build-clang/build-clang.py'
             - 'build/build-clang/clang-trunk-mingw.json'
             - 'taskcluster/scripts/misc/build-clang-trunk-mingw.sh'
         toolchain-artifact: public/build/fxc2.tar.xz
     toolchains:
         - linux64-clang-trunk-mingw-x86
 
+linux64-custom-v8:
+    description: "Custom v8 build"
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        symbol: TL(custom-v8)
+        tier: 1
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        docker-image: {in-tree: custom-v8}
+        max-run-time: 3600
+    run:
+        using: toolchain-script
+        script: build-custom-v8.sh
+        arguments: [
+            d8.zip,
+            is_debug=false,
+            target_cpu="x64"
+        ]
+        toolchain-artifact: public/build/d8.zip
+
 wrench-deps:
     description: "Downloads all the crates needed for building wrench"
     treeherder:
         kind: build
         platform: toolchains/opt
         symbol: WR(wrench-deps)
         tier: 2
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
new file mode 100644
--- /dev/null
+++ b/taskcluster/docker/custom-v8/Dockerfile
@@ -0,0 +1,24 @@
+# %ARG DOCKER_IMAGE_PARENT
+FROM $DOCKER_IMAGE_PARENT
+MAINTAINER Joel Maher <jmaher@mozilla.com>
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/workspace
+VOLUME /builds/worker/tooltool-cache
+
+ENV XZ_OPT=-T0
+
+RUN apt-get update && \
+    apt-get install \
+      bzip2 \
+      curl \
+      libbz2-dev \
+      lsb-core \
+      sudo \
+      tar \
+      unzip \
+      wget \
+      zip
+
+RUN adduser worker sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/misc/build-custom-v8.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+set -x -e -v
+
+# This script is for building a custom version of V8
+ARTIFACT_NAME='d8.zip'
+CONFIG='is_debug=false target_cpu="x64"'
+if [[ $# -eq 0 ]]; then
+    echo "Using default configuration for v8 build."
+    CONFIG=$(echo $CONFIG | tr -d "'")
+else
+	# First argument must be the artifact name
+	ARTIFACT_NAME="$1"
+	shift
+
+	# Use the rest of the arguments as the build config
+	CONFIG=$(echo $* | tr -d "'")
+fi
+
+echo "Config: $CONFIG"
+echo "Artifact name: $ARTIFACT_NAME"
+
+WORKSPACE=$HOME/workspace
+HOME_DIR=$WORKSPACE/build
+UPLOAD_DIR=$HOME/artifacts
+
+cd $HOME_DIR/src
+
+# Setup depot_tools
+git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
+export PATH=$PATH:$HOME_DIR/src/depot_tools
+gclient
+
+# Get v8 source code
+mkdir v8
+cd v8
+fetch v8
+cd v8
+
+# Build dependencies
+gclient sync
+./build/install-build-deps.sh
+
+# Build v8
+git checkout master
+git pull && gclient sync
+
+gn gen out/release --args="$CONFIG"
+ninja -C out/release d8
+
+# Gather binary and related files into a zip, and upload it
+cd ..
+mkdir d8
+
+cp -R v8/out/release d8
+cp -R v8/include d8
+chmod -R +x d8
+
+zip -r $ARTIFACT_NAME d8
+
+mkdir -p $UPLOAD_DIR
+cp $ARTIFACT_NAME $UPLOAD_DIR