Bug 1630009: Update conditioned profile build dependencies; r=tarek
authorTom Prince <mozilla@hocat.ca>
Fri, 17 Apr 2020 07:00:48 +0000
changeset 524553 458f9c1098bced8d78d106d0b8ff958bd1e2531d
parent 524552 40177ff41c87dea585f9c428882d4e7efd8c96c3
child 524554 bf118a2f538a7e0c203cb63710edc6f6a77617f1
push id37323
push userdluca@mozilla.com
push dateFri, 17 Apr 2020 16:25:55 +0000
treeherdermozilla-central@b4b1d6f91ef0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstarek
bugs1630009
milestone77.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1630009: Update conditioned profile build dependencies; r=tarek This make four changes: - use shippable/pgo builds on win64 and android platforms - use linux64 instead of linux to get test packages for out-of-tree tasks - consistently use `build` as the dependency name in tasks - use the geckodriver toolchain, rather than the one packed in tasks Differential Revision: https://phabricator.services.mozilla.com/D70943
taskcluster/ci/condprof/kind.yml
--- a/taskcluster/ci/condprof/kind.yml
+++ b/taskcluster/ci/condprof/kind.yml
@@ -1,301 +1,317 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 loader: taskgraph.loader.transform:loader
 
 kind-dependencies:
+    - toolchain
     - build
 
 transforms:
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
 job-defaults:
     run-on-projects:
         - try
         - mozilla-central
     treeherder:
         kind: other
         tier: 2
     run:
         using: run-task
+        cwd: fetches/condprofile
         checkout: false
         tooltool-downloads: public
     scopes:
         - secrets:get:project/releng/gecko/build/level-{level}/conditioned-profiles
     worker:
         taskcluster-proxy: true
         max-run-time: 10800
-        env:
-            MOZ_FETCHES_DIR: fetches
-            CONDPROF_ARCHIVES: /builds/worker/archive
-            CONDPROF_ROOT: fetches/condprofile
 jobs:
     windows2012-64-firefox:
         worker-type: b-win2012
         description: Creates or updates conditioned profiles on Win64
         treeherder:
             symbol: condprof(firefox)
             platform: windows2012-64/opt
         index:
             product: firefox
             job-name: condprof-win64
         dependencies:
-            build-win64/opt: build-win64/opt
+            build: build-win64-shippable/opt
         fetches:
-            build-win64/opt:
-                - artifact: target.common.tests.tar.gz
+            build:
                 - artifact: target.condprof.tests.tar.gz
                 - artifact: target.zip
+            toolchain:
+                - win64-geckodriver
         run:
-            using: run-task
             command: >-
-                cd %CONDPROF_ROOT% &&
-                python3.exe condprof/main.py --geckodriver z:/%USERNAME%/fetches/bin/geckodriver.exe --firefox firefox/firefox.exe z:/%USERNAME%/archive
+                python3.exe condprof/main.py
+                --geckodriver ${MOZ_FETCHES_DIR}/geckodriver.exe
+                --firefox ${MOZ_FETCHES_DIR}/firefox/firefox.exe
+                ../../archive
         worker:
             artifacts:
                 - name: public/condprof
                   path: archive
                   type: directory
     linux64-firefox:
         worker-type: b-linux
         description: Creates or updates conditioned profiles on Linux
         treeherder:
             symbol: condprof(firefox)
             platform: linux64/opt
         index:
             product: firefox
             job-name: condprof-linux64
         dependencies:
-            build-linux/opt: build-linux-shippable/opt
+            build: build-linux64-shippable/opt
         fetches:
-            build-linux/opt:
-                - artifact: target.common.tests.tar.gz
+            build:
                 - artifact: target.condprof.tests.tar.gz
                 - artifact: target.tar.bz2
+            toolchain:
+                - linux64-geckodriver
         run:
-            using: run-task
             command: >-
-                $CONDPROF_ROOT/virtualenv/virtualenv.py -p python3.7 $CONDPROF_ROOT &&
-                cd $CONDPROF_ROOT &&
-                export PATH="$PATH:$MOZ_FETCHES_DIR/bin" &&
-                bin/python3.7 condprof/main.py --firefox $MOZ_FETCHES_DIR/firefox/firefox-bin $CONDPROF_ARCHIVES
+                virtualenv/virtualenv.py -p python3.7 . &&
+                bin/python3.7 condprof/main.py
+                --geckodriver ${MOZ_FETCHES_DIR}/geckodriver
+                --firefox ${MOZ_FETCHES_DIR}/firefox/firefox-bin
+                ../../archive
         worker:
             docker-image: {in-tree: condprof}
             artifacts:
                 - name: public/condprof
                   path: /builds/worker/archive
                   type: directory
     macosx64-firefox:
         worker-type: t-osx-1014
         description: Creates or updates conditioned profiles on macOS
         treeherder:
             symbol: condprof(firefox)
             platform: macosx1014-64/opt
-        dependencies:
-            build-macosx64/opt: build-macosx64-shippable/opt
-        fetches:
-            build-macosx64/opt:
-                - artifact: target.common.tests.tar.gz
-                - artifact: target.condprof.tests.tar.gz
-                - artifact: target.dmg
         index:
             product: firefox
             job-name: condprof-macosx64
+        dependencies:
+            build: build-macosx64-shippable/opt
+        fetches:
+            build:
+                - artifact: target.condprof.tests.tar.gz
+                - artifact: target.dmg
+                  extract: false
+            toolchain:
+                - macosx64-geckodriver
         run:
-            using: run-task
             command: >-
-                $CONDPROF_ROOT/virtualenv/virtualenv.py -p python3 $CONDPROF_ROOT &&
-                export PATH="$PATH:$MOZ_FETCHES_DIR/bin" &&
-                cd $CONDPROF_ROOT &&
-                bin/python3 condprof/main.py --firefox $MOZ_FETCHES_DIR/target.dmg ../../archive
+                virtualenv/virtualenv.py -p python3 . &&
+                bin/python3 condprof/main.py
+                --geckodriver ${MOZ_FETCHES_DIR}/geckodriver
+                --firefox ${MOZ_FETCHES_DIR}/target.dmg
+                ../../archive
         worker:
             artifacts:
                 - name: public/condprof
                   path: archive
                   type: directory
     android-hw-g5-7-0-arm7-api-16-geckoview:
         worker-type: t-bitbar-gw-perf-g5
         description: Creates or update conditioned profiles on GeckoView+G5
         treeherder:
             symbol: condprof(geckoview)
             platform: android-hw-g5-7-0-arm7-api-16/opt
         index:
             product: firefox
             job-name: condprof-g5-geckoview_example
         dependencies:
-            build-linux/opt: build-linux-shippable/opt
-            build-android-api-16/opt: build-android-api-16/opt
+            build: build-android-api-16/pgo
         fetches:
-            build-linux/opt:
-                - artifact: target.common.tests.tar.gz
+            build:
                 - artifact: target.condprof.tests.tar.gz
-            build-android-api-16/opt:
                 - artifact: geckoview_example.apk
+                  extract: false
+            toolchain:
+                - linux64-geckodriver
         run:
             run-as-root: true
-            using: run-task
             command: >-
                 adb install -r $MOZ_FETCHES_DIR/geckoview_example.apk &&
-                $CONDPROF_ROOT/virtualenv/virtualenv.py -p python3 $CONDPROF_ROOT &&
-                cd $CONDPROF_ROOT &&
-                export PATH="$PATH:$MOZ_FETCHES_DIR/bin" &&
-                bin/python3 condprof/main.py --device-name g5 --firefox org.mozilla.geckoview_example ../../archive
+                virtualenv/virtualenv.py -p python3 . &&
+                bin/python3 condprof/main.py
+                --geckodriver ${MOZ_FETCHES_DIR}/geckodriver
+                --device-name g5
+                --firefox org.mozilla.geckoview_example
+                ../../archive
         worker:
             artifacts:
                 - name: public/condprof
                   path: archive
                   type: directory
     android-hw-g5-7-0-arm7-api-16-fenix:
         worker-type: t-bitbar-gw-perf-g5
         description: Creates or update conditioned profiles on Fenix+G5
         treeherder:
             symbol: condprof(fenix)
             platform: android-hw-g5-7-0-arm7-api-16/opt
         index:
             product: firefox
             job-name: condprof-g5-fenix.performancetest
         dependencies:
-            build-linux/opt: build-linux-shippable/opt
+            build: build-linux64-shippable/opt
         fetches:
-            build-linux/opt:
-                - artifact: target.common.tests.tar.gz
+            build:
                 - artifact: target.condprof.tests.tar.gz
+            toolchain:
+                - linux64-geckodriver
         run:
             run-as-root: true
             command: >-
                 curl -L -o target.apk https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/project.mobile.fenix.v2.performance-test.latest/artifacts/public/build/armeabi-v7a/geckoNightly/target.apk &&
                 adb install -r target.apk &&
                 rm -rf target.apk &&
-                $CONDPROF_ROOT/virtualenv/virtualenv.py -p python3 $CONDPROF_ROOT &&
-                cd $CONDPROF_ROOT &&
-                export PATH="$PATH:$MOZ_FETCHES_DIR/bin" &&
-                bin/python3 condprof/main.py --device-name g5 --firefox org.mozilla.fenix.performancetest ../../archive
+                virtualenv/virtualenv.py -p python3 . &&
+                bin/python3 condprof/main.py
+                --geckodriver ${MOZ_FETCHES_DIR}/geckodriver
+                --device-name g5
+                --firefox org.mozilla.fenix.performancetest
+                ../../archive
         worker:
             artifacts:
                 - name: public/condprof
                   path: archive
                   type: directory
     android-hw-g5-7-0-arm7-api-16-fennec68:
         worker-type: t-bitbar-gw-perf-g5
         description: Creates or update conditioned profiles on Fennec68+G5
         treeherder:
             symbol: condprof(fennec68)
             platform: android-hw-g5-7-0-arm7-api-16/opt
         index:
             product: firefox
             job-name: condprof-g5-firefox
         dependencies:
-            build-linux/opt: build-linux-shippable/opt
+            build: build-linux64-shippable/opt
         fetches:
-            build-linux/opt:
-                - artifact: target.common.tests.tar.gz
+            build:
                 - artifact: target.condprof.tests.tar.gz
+            toolchain:
+                - linux64-geckodriver
         run:
             run-as-root: true
             command: >-
                 curl -L -o target.apk https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-esr68.signed-nightly.nightly.latest.mobile.android-api-16-release-opt/artifacts/public/build/target.apk &&
                 adb install -r target.apk &&
-                rm -rf target.apk &&
-                $CONDPROF_ROOT/virtualenv/virtualenv.py -p python3 $CONDPROF_ROOT &&
-                cd $CONDPROF_ROOT &&
-                export PATH="$PATH:$MOZ_FETCHES_DIR/bin" &&
-                bin/python3 condprof/main.py --device-name g5 --firefox org.mozilla.firefox ../../archive
+                virtualenv/virtualenv.py -p python3 . &&
+                bin/python3 condprof/main.py
+                --geckodriver ${MOZ_FETCHES_DIR}/geckodriver
+                --device-name g5
+                --firefox org.mozilla.firefox
+                ../../archive
         worker:
             artifacts:
                 - name: public/condprof
                   path: archive
                   type: directory
     android-hw-p2-8-0-aarch64-fenix:
         worker-type: t-bitbar-gw-perf-p2
         description: Creates or update conditioned profiles on Fenix+P2
         treeherder:
             symbol: condprof(fenix)
             platform: android-hw-p2-8-0-android-aarch64/opt
-        dependencies:
-            build-linux/opt: build-linux-shippable/opt
-        fetches:
-            build-linux/opt:
-                - artifact: target.common.tests.tar.gz
-                - artifact: target.condprof.tests.tar.gz
         index:
             product: firefox
             job-name: condprof-p2_aarch64-fenix.performancetest
+        dependencies:
+            build: build-android-api-16/pgo
+        fetches:
+            build:
+                - artifact: target.condprof.tests.tar.gz
+            toolchain:
+                - linux64-geckodriver
         run:
             run-as-root: true
             command: >-
                 curl -L -o target.apk https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/project.mobile.fenix.v2.performance-test.latest/artifacts/public/build/arm64-v8a/geckoNightly/target.apk &&
                 adb install -r target.apk &&
                 rm -rf target.apk &&
-                $CONDPROF_ROOT/virtualenv/virtualenv.py -p python3 $CONDPROF_ROOT &&
-                cd $CONDPROF_ROOT &&
-                export PATH="$PATH:$MOZ_FETCHES_DIR/bin" &&
-                bin/python3 condprof/main.py --device-name p2_aarch64 --firefox org.mozilla.fenix.performancetest ../../archive
+                virtualenv/virtualenv.py -p python3 . &&
+                bin/python3 condprof/main.py
+                --geckodriver ${MOZ_FETCHES_DIR}/geckodriver
+                --device-name p2_aarch64
+                --firefox org.mozilla.fenix.performancetest
+                ../../archive
         worker:
             artifacts:
                 - name: public/condprof
                   path: archive
                   type: directory
     android-hw-p2-8-0-android-aarch64-geckoview:
         worker-type: t-bitbar-gw-perf-p2
         description: Creates or update conditioned profiles on GeckoView+P2
         treeherder:
             symbol: condprof(geckoview)
             platform: android-hw-p2-8-0-android-aarch64/opt
-        dependencies:
-            build-linux/opt: build-linux-shippable/opt
-            build-android-api-16/opt: build-android-api-16/opt
         index:
             product: firefox
             job-name: condprof-p2_aarch64-geckoview_example
+        dependencies:
+            build: build-android-api-16/pgo
         fetches:
-            build-linux/opt:
-                - artifact: target.common.tests.tar.gz
+            build:
                 - artifact: target.condprof.tests.tar.gz
-            build-android-api-16/opt:
                 - artifact: geckoview_example.apk
+                  extract: false
+            toolchain:
+                - linux64-geckodriver
         run:
             run-as-root: true
-            using: run-task
             command: >-
                 adb install -r $MOZ_FETCHES_DIR/geckoview_example.apk &&
-                $CONDPROF_ROOT/virtualenv/virtualenv.py -p python3 $CONDPROF_ROOT &&
-                cd $CONDPROF_ROOT &&
-                export PATH="$PATH:$MOZ_FETCHES_DIR/bin" &&
-                bin/python3 condprof/main.py --device-name p2_aarch64 --firefox org.mozilla.geckoview_example ../../archive
+                virtualenv/virtualenv.py -p python3 . &&
+                bin/python3 condprof/main.py
+                --geckodriver ${MOZ_FETCHES_DIR}/geckodriver
+                --device-name p2_aarch64
+                --firefox org.mozilla.geckoview_example
+                ../../archive
         worker:
             artifacts:
                 - name: public/condprof
                   path: archive
                   type: directory
     android-hw-p2-8-0-aarch64-fennec68:
         worker-type: t-bitbar-gw-perf-p2
         description: Creates or update conditioned profiles on Fennec68+P2
         treeherder:
             symbol: condprof(fennec68)
             platform: android-hw-p2-8-0-android-aarch64/opt
-        dependencies:
-            build-linux/opt: build-linux-shippable/opt
-        fetches:
-            build-linux/opt:
-                - artifact: target.common.tests.tar.gz
-                - artifact: target.condprof.tests.tar.gz
         index:
             product: firefox
             job-name: condprof-p2_aarch64-firefox
+        dependencies:
+            build: build-linux64-shippable/opt
+        fetches:
+            build:
+                - artifact: target.condprof.tests.tar.gz
+            toolchain:
+                - linux64-geckodriver
         run:
             run-as-root: true
             command: >-
                 curl -L -o target.apk https://firefox-ci-tc.services.mozilla.com/api/index/v1/task/gecko.v2.mozilla-esr68.signed-nightly.nightly.latest.mobile.android-aarch64-release-opt/artifacts/public/build/target.apk &&
                 adb install -r target.apk &&
                 rm -rf target.apk &&
-                $CONDPROF_ROOT/virtualenv/virtualenv.py -p python3 $CONDPROF_ROOT &&
-                cd $CONDPROF_ROOT &&
-                export PATH="$PATH:$MOZ_FETCHES_DIR/bin" &&
-                bin/python3 condprof/main.py --device-name p2_aarch64 --firefox org.mozilla.firefox ../../archive
+                virtualenv/virtualenv.py -p python3 . &&
+                bin/python3 condprof/main.py
+                --geckodriver ${MOZ_FETCHES_DIR}/geckodriver
+                --device-name p2_aarch64
+                --firefox org.mozilla.firefox
+                ../../archive
         worker:
             artifacts:
                 - name: public/condprof
                   path: archive
                   type: directory