Bug 1700774 - [taskgraph] Group 'raptor' specific test configs together, r=perftest-reviewers,sparky
authorAndrew Halberstadt <ahal@mozilla.com>
Mon, 29 Nov 2021 15:52:20 +0000
changeset 600383 0d0151af2eceb0db881e9dc9c509a50acc75c599
parent 600382 197932d5aecb1acade386773cf660c28ff0818e5
child 600384 05210e8d9e8d1145641b0122eeff84367973a551
push id39021
push userncsoregi@mozilla.com
push dateMon, 29 Nov 2021 21:53:24 +0000
treeherdermozilla-central@408b89a2e02c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest-reviewers, sparky
bugs1700774
milestone96.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1700774 - [taskgraph] Group 'raptor' specific test configs together, r=perftest-reviewers,sparky In the test transforms we currently have general 'test_description_schema' and a schema for raptor / browsertime tasks. Prior, the raptor schema was validated *before* the test schema, and we were doing a lot of validation twice. This revision switches the order such that the test schema is first evaluated, and then the raptor one is. To accomplish this, we create a 'raptor' subconfig that the 'test_description_schema' completely ignores (and leaves up to the raptor schema to evaluate). The benefit of this change is that all test tasks will have a single entry point when they get into the 'test' transforms. This makes reasoning about their configuration much easier and lays the ground work to start splitting even more transforms out into other modules. Differential Revision: https://phabricator.services.mozilla.com/D132069
taskcluster/ci/test/browsertime-desktop.yml
taskcluster/ci/test/browsertime-mobile.yml
taskcluster/gecko_taskgraph/transforms/test/__init__.py
taskcluster/gecko_taskgraph/transforms/test/raptor.py
--- a/taskcluster/ci/test/browsertime-desktop.yml
+++ b/taskcluster/ci/test/browsertime-desktop.yml
@@ -34,17 +34,16 @@ job-defaults:
                     - raptor/linux_config.py
         extra-options:
             - --browsertime
             - --conditioned-profile=settled
     python-3: true
 
 browsertime-tp6:
     <<: &tp6-defaults
-        apps: [firefox, chrome, chromium]
         description: Raptor (browsertime) tp6 page-load tests
         limit-platforms:
             by-app:
                 chrome:
                     - .*shippable.*
                 chromium:
                     - .*shippable.*
                 default: []
@@ -56,17 +55,16 @@ browsertime-tp6:
                         - --chimera
                         - --browsertime
                         - --browsertime-no-ffwindowrecorder
                         - --conditioned-profile=settled
                     default:
                         - --chimera
                         - --browsertime
                         - --conditioned-profile=settled
-        raptor-test: tp6
         variants: [fission]
         run-on-projects:
             by-variant:
                 fission:
                     by-test-platform:
                         windows.*-32.*: []
                         .*shippable-qr/.*: [mozilla-central]
                         default: []
@@ -76,59 +74,66 @@ browsertime-tp6:
                             by-test-platform:
                                 (linux|windows|macos)(?!.*-qr).*: []
                                 linux.*shippable.*: [trunk, mozilla-beta]
                                 macosx1014-64-shippable-qr.*: [trunk]
                                 macosx1015.*shippable-qr.*: [trunk, mozilla-beta]
                                 windows10-64.*shippable.*: [trunk, mozilla-beta]
                                 default: []
                         default: []
+        treeherder-symbol: Btime(tp6)
+    raptor:
+        apps: [firefox, chrome, chromium]
+        test: tp6
         run-visual-metrics: true
-        treeherder-symbol: Btime(tp6)
-    raptor-subtests:
-        - buzzfeed
-        - ebay
-        - espn
-        - expedia
-        - [facebook, fb]
-        - [google-docs, gdocs]
-        - [google-docs-canvas, gcanvas]
-        - [google-mail, gmail]
-        - [google-search, gsearch]
-        - imdb
-        - imgur
-        - linkedin
-        - microsoft
-        - netflix
-        - nytimes
-        - office
-        - outlook
-        - paypal
-        - pinterest
-        - reddit
-        - tumblr
-        - twitch
-        - wikia
-        - youtube
+        subtests:
+            - buzzfeed
+            - ebay
+            - espn
+            - expedia
+            - [facebook, fb]
+            - [google-docs, gdocs]
+            - [google-docs-canvas, gcanvas]
+            - [google-mail, gmail]
+            - [google-search, gsearch]
+            - imdb
+            - imgur
+            - linkedin
+            - microsoft
+            - netflix
+            - nytimes
+            - office
+            - outlook
+            - paypal
+            - pinterest
+            - reddit
+            - tumblr
+            - twitch
+            - wikia
+            - youtube
     tier:
         by-variant:
             fission: 2
             default:
                 by-app:
                     firefox: 2
                     default: 3
 
 browsertime-responsiveness:
     <<: *tp6-defaults
-    raptor-subtests:
-        - cnn-nav
-        - [facebook-nav, fbnav]
-        - [reddit-billgates-ama, rdt-ama]
-        - [reddit-billgates-post-1, rdt-post-1]
-        - [reddit-billgates-post-2, rdt-post-2]
+    raptor:
+        apps: [firefox, chrome, chromium]
+        test: tp6
+        run-visual-metrics: true
+        subtests:
+            - cnn-nav
+            - [facebook-nav, fbnav]
+            - [reddit-billgates-ama, rdt-ama]
+            - [reddit-billgates-post-1, rdt-post-1]
+            - [reddit-billgates-post-2, rdt-post-2]
     mozharness:
         extra-options:
             by-test-platform:
                 windows10-.*:
                     - --cold
                     - --browsertime
                     - --browsertime-no-ffwindowrecorder
                     - --conditioned-profile=settled
@@ -155,26 +160,30 @@ browsertime-responsiveness:
                     default: []
     tier:
         by-app:
             firefox: 2
             default: 3
 
 browsertime-tp6-essential:
     <<: *tp6-defaults
-    raptor-subtests:
-        - amazon
-        - [bing-search, bing]
-        - cnn
-        - fandom
-        - [google-slides, gslides]
-        - instagram
-        - twitter
-        - wikipedia
-        - yahoo-mail
+    raptor:
+        apps: [firefox, chrome, chromium]
+        test: tp6
+        run-visual-metrics: true
+        subtests:
+            - amazon
+            - [bing-search, bing]
+            - cnn
+            - fandom
+            - [google-slides, gslides]
+            - instagram
+            - twitter
+            - wikipedia
+            - yahoo-mail
     run-on-projects:
         by-app:
             firefox:
                 by-test-platform:
                     linux.*shippable.*: [trunk, mozilla-beta]
                     macosx1014-64-shippable-qr.*: [trunk]
                     macosx1015.*shippable.*: [trunk, mozilla-beta]
                     windows10-64.*shippable.*: [trunk, mozilla-beta]
@@ -186,28 +195,29 @@ browsertime-tp6-essential:
             linux1804-64-clang-trunk-qr/opt: 2
             default:
                 by-app:
                     firefox: 1
                     default: 3
 
 browsertime-benchmark:
     description: Raptor (browsertime) Benchmark tests
-    raptor-subtests:
-        - ares6
-        - [assorted-dom, dom]
-        - [jetstream2, js2]
-        - [motionmark-animometer, mm-a]
-        - [motionmark-htmlsuite, mm-h]
-        - [speedometer, sp]
-        - [stylebench, sb]
-        - [sunspider, ss]
-        - [unity-webgl, ugl]
-        - [webaudio, wa]
-    apps: [firefox, chrome, chromium]
+    raptor:
+        apps: [firefox, chrome, chromium]
+        subtests:
+            - ares6
+            - [assorted-dom, dom]
+            - [jetstream2, js2]
+            - [motionmark-animometer, mm-a]
+            - [motionmark-htmlsuite, mm-h]
+            - [speedometer, sp]
+            - [stylebench, sb]
+            - [sunspider, ss]
+            - [unity-webgl, ugl]
+            - [webaudio, wa]
     treeherder-symbol: Btime()
     tier:
         by-variant:
             fission: 3
             default:
                 by-app:
                     firefox:
                         by-subtest:
@@ -305,29 +315,30 @@ browsertime-benchmark:
     fetches:
         fetch:
             - assorted-dom
             - jetstream2
             - unity-webgl
 
 browsertime-benchmark-wasm:
     description: Raptor (browsertime) WASM benchmarks
-    raptor-subtests:
-        by-app:
-            firefox:
-                - [wasm-godot, godot]
-                - [wasm-godot-baseline, godot-b]
-                - [wasm-godot-optimizing, godot-i]
-                - [wasm-misc, wm]
-                - [wasm-misc-baseline, wm-b]
-                - [wasm-misc-optimizing, wm-i]
-            default:
-                - [wasm-misc, wm]
-                - [wasm-godot, godot]
-    apps: [firefox, chrome, chromium]
+    raptor:
+        apps: [firefox, chrome, chromium]
+        subtests:
+            by-app:
+                firefox:
+                    - [wasm-godot, godot]
+                    - [wasm-godot-baseline, godot-b]
+                    - [wasm-godot-optimizing, godot-i]
+                    - [wasm-misc, wm]
+                    - [wasm-misc-baseline, wm-b]
+                    - [wasm-misc-optimizing, wm-i]
+                default:
+                    - [wasm-misc, wm]
+                    - [wasm-godot, godot]
     treeherder-symbol: Btime()
     tier: 2
     variants: [fission]
     run-on-projects:
         by-variant:
             fission:
                 by-test-platform:
                     windows.*-32.*: []
@@ -352,27 +363,28 @@ browsertime-benchmark-wasm:
             wasm-godot-baseline: 1500
             default: 900
     fetches:
         fetch:
             - wasm-misc
 
 browsertime-youtube-playback:
     description: Browsertime YouTube Playback on Firefox
-    raptor-subtests:
-        by-app:
-            firefox:
-                - [youtube-playback-av1-sfr, ytp-av1-sfr]
-                - [youtube-playback-h264-sfr, ytp-h264-sfr]
-                - [youtube-playback-hfr, ytp-hfr]
-                - [youtube-playback-vp9-sfr, ytp-vp9-sfr]
-                - [youtube-playback-widevine-hfr, ytp-widevine-hfr]
-                - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr]
-                - [youtube-playback-widevine-vp9-sfr, ytp-widevine-vp9-sfr]
-    apps: [firefox]
+    raptor:
+        apps: [firefox]
+        subtests:
+            by-app:
+                firefox:
+                    - [youtube-playback-av1-sfr, ytp-av1-sfr]
+                    - [youtube-playback-h264-sfr, ytp-h264-sfr]
+                    - [youtube-playback-hfr, ytp-hfr]
+                    - [youtube-playback-vp9-sfr, ytp-vp9-sfr]
+                    - [youtube-playback-widevine-hfr, ytp-widevine-hfr]
+                    - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr]
+                    - [youtube-playback-widevine-vp9-sfr, ytp-widevine-vp9-sfr]
     treeherder-symbol: Btime()
     variants: [fission]
     run-on-projects:
         by-variant:
             fission:
                 by-test-platform:
                     windows.*-32.*: []
                     .*shippable-qr/.*: [mozilla-central]
@@ -400,127 +412,139 @@ browsertime-youtube-playback:
 browsertime-tp6-live:
     <<: *tp6-defaults
     description: Raptor (browsertime) tp6 on live-sites
     mozharness:
         extra-options:
             - --browser-cycles=15
             - --chimera
             - --live-sites
-    raptor-subtests:
-        - amazon
-        - [bing-search, bing]
-        - ebay
-        - [facebook, fb]
-        - fandom
-        - [google-docs, gdocs]
-        - [google-mail, gmail]
-        - [google-search, gsearch]
-        - [google-slides, gslides]
-        - imdb
-        - imgur
-        - instagram
-        - linkedin
-        - microsoft
-        - netflix
-        # - office (site loads blank page if not signed in)
-        - outlook
-        - paypal
-        - pinterest
-        - reddit
-        - tumblr
-        - twitch
-        - twitter
-        - wikipedia
-        - yahoo-mail
-        - youtube
+    raptor:
+        apps: [firefox, chrome, chromium]
+        test: tp6
+        run-visual-metrics: true
+        subtests:
+            - amazon
+            - [bing-search, bing]
+            - ebay
+            - [facebook, fb]
+            - fandom
+            - [google-docs, gdocs]
+            - [google-mail, gmail]
+            - [google-search, gsearch]
+            - [google-slides, gslides]
+            - imdb
+            - imgur
+            - instagram
+            - linkedin
+            - microsoft
+            - netflix
+            # - office (site loads blank page if not signed in)
+            - outlook
+            - paypal
+            - pinterest
+            - reddit
+            - tumblr
+            - twitch
+            - twitter
+            - wikipedia
+            - yahoo-mail
+            - youtube
     run-on-projects: []
     tier: 3
     treeherder-symbol: Btime-live(tp6)
 
 browsertime-tp6-live-sheriffed:
     <<: *tp6-defaults
-    apps: [firefox]
+    raptor:
+        apps: [firefox]
+        test: tp6
+        run-visual-metrics: true
+        subtests:
+            - cnn
     description: Raptor (browsertime) tp6 on live-sites
     mozharness:
         extra-options:
             - --browser-cycles=5
             - --chimera
             - --live-sites
-    raptor-subtests:
-        - cnn
     run-on-projects:
         by-variant:
             fission: []
             default:
                 by-test-platform:
                     (linux|windows10-64|macos)(?!.*shippable).*: []
                     (linux|windows|macosx1015)(?!.*-qr).*: []
                     windows.*-32.*: []
                     macosx1014.*: []
                     default: []
     tier: 2
     treeherder-symbol: Btime-live(tp6)
 
 browsertime-tp6-profiling:
     <<: *tp6-defaults
-    apps: [firefox]
+    raptor:
+        apps: [firefox]
+        test: tp6
+        run-visual-metrics: true
+        subtests: [amazon]
     description: Raptor (browsertime) tp6 page-load tests with Gecko Profiling
     mozharness:
         extra-options:
             - --chimera
             - --gecko-profile
-    raptor-subtests: [amazon]
     run-on-projects:
         by-variant:
             fission: []
             default:
                 by-test-platform:
                     (linux|windows|macosx1015)(?!.*-qr).*: []
                     (linux|windows|macosx1015).*shippable-qr.*: [mozilla-central]
                     default: []
     tier:
         by-variant:
             fission: 2
             default: 3
     treeherder-symbol: Btime-Prof(tp6)
 
 browsertime-custom:
-    apps: [firefox]
+    raptor:
+        apps: [firefox]
+        run-visual-metrics: true
+        subtests: [[process-switch, ps]]
     description: Raptor (browsertime) custom pageload tests
-    run-visual-metrics: true
     max-run-time: 2700
     mozharness:
         extra-options:
             - --chimera
-    raptor-subtests: [[process-switch, ps]]
     variants: [fission]
     run-on-projects:
         by-variant:
             fission:
                 by-test-platform:
                     windows.*-32.*: []
                     .*shippable-qr/.*: [mozilla-central]
                     default: []
             default:
                 by-test-platform:
                     (linux|windows10-64|macosx).*shippable-qr.*: [mozilla-central]
                     default: []
     tier: 2
     treeherder-symbol: Btime(ps)
 
 browsertime-first-install:
-    apps: [firefox]
+    raptor:
+        apps: [firefox]
+        run-visual-metrics: true
+        subtests: [welcome]
     description: Raptor (browsertime) custom pageload tests
-    run-visual-metrics: true
     max-run-time: 2700
     mozharness:
         extra-options:
             - --chimera
-    raptor-subtests: [welcome]
     variants: [fission]
     run-on-projects:
         by-variant:
             fission:
                 by-test-platform:
                     windows.*-32.*: []
                     .*shippable-qr/.*: [mozilla-central]
                     default: []
--- a/taskcluster/ci/test/browsertime-mobile.yml
+++ b/taskcluster/ci/test/browsertime-mobile.yml
@@ -30,28 +30,29 @@ job-defaults:
                         index: mobile.v2.reference-browser.raptor.latest.arm64-v8a
                         name: target.arm64-v8a.apk
                     default:
                         index: mobile.v2.reference-browser.raptor.latest.armeabi-v7a
                         name: target.armeabi-v7a.apk
             default: geckoview_example.apk
     e10s: true
     virtualization: hardware
-    activity:
-        by-app:
-            fenix: org.mozilla.fenix.IntentReceiverActivity
-            geckoview: org.mozilla.geckoview_example.GeckoViewActivity
-            refbrow: org.mozilla.reference.browser.BrowserTestActivity
-            default: None
-    binary-path:
-        by-app:
-            fenix: org.mozilla.fenix
-            geckoview: org.mozilla.geckoview_example
-            refbrow: org.mozilla.reference.browser.raptor
-            default: None
+    raptor:
+        activity:
+            by-app:
+                fenix: org.mozilla.fenix.IntentReceiverActivity
+                geckoview: org.mozilla.geckoview_example.GeckoViewActivity
+                refbrow: org.mozilla.reference.browser.BrowserTestActivity
+                default: None
+        binary-path:
+            by-app:
+                fenix: org.mozilla.fenix
+                geckoview: org.mozilla.geckoview_example
+                refbrow: org.mozilla.reference.browser.raptor
+                default: None
     mozharness:
         script: raptor_script.py
         config:
             - raptor/android_hw_config.py
         extra-options:
             - --browsertime
             # Bug 1635749, disable window recorder temporarily
             - --browsertime-no-ffwindowrecorder
@@ -62,69 +63,74 @@ job-defaults:
     limit-platforms:
         by-app:
             chrome-m:
                 - .*shippable-qr.*
             default: []
 
 browsertime-tp6m:
     <<: &tp6m-defaults
-        apps: [geckoview, fenix, chrome-m, refbrow]
         description: Raptor (browsertime) tp6 page-load tests on android
         mozharness:
             extra-options:
                 - --chimera
-        raptor-test: tp6m
-        run-visual-metrics: true
         tier:
             by-app:
                 geckoview: 1
                 refbrow: 2
                 default: 3
         treeherder-symbol: Btime(tp6m)
-    raptor-subtests:
-        - amazon
-        - bing
-        - [bing-search-restaurants, bing-s-r]
-        - booking
-        - cnn
-        - [cnn-ampstories, cnn-amp]
-        - dailymail
-        - [ebay-kleinanzeigen, ebay-k]
-        - [ebay-kleinanzeigen-search, ebay-k-s]
-        - [facebook-cristiano, fb-cris]
-        - [google-maps, gmaps]
-        - [google-search-restaurants, gsearch-r]
-        - instagram
-        - imdb
-        - reddit
-        - sina
-        - [stackoverflow, stacko]
-        - web-de
-        - wikipedia
-        - youtube
+    raptor:
+        apps: [geckoview, fenix, chrome-m, refbrow]
+        run-visual-metrics: true
+        test: tp6m
+        subtests:
+            - amazon
+            - bing
+            - [bing-search-restaurants, bing-s-r]
+            - booking
+            - cnn
+            - [cnn-ampstories, cnn-amp]
+            - dailymail
+            - [ebay-kleinanzeigen, ebay-k]
+            - [ebay-kleinanzeigen-search, ebay-k-s]
+            - [facebook-cristiano, fb-cris]
+            - [google-maps, gmaps]
+            - [google-search-restaurants, gsearch-r]
+            - instagram
+            - imdb
+            - reddit
+            - sina
+            - [stackoverflow, stacko]
+            - web-de
+            - wikipedia
+            - youtube
     run-on-projects:
         by-app:
             geckoview:
                 by-test-platform:
                     android-hw-g5.*shippable-qr.*: [trunk, mozilla-beta]
                     android-hw-p2.*aarch64-shippable-qr.*: [trunk, mozilla-beta]
                     default: []
             default: []
 
 browsertime-tp6m-essential:
     <<: *tp6m-defaults
-    raptor-subtests:
-        - allrecipes
-        - [amazon-search, amazon-s]
-        - espn
-        - facebook
-        - google
-        - [microsoft-support, micros-sup]
-        - [youtube-watch, youtube-w]
+    raptor:
+        apps: [geckoview, fenix, chrome-m, refbrow]
+        run-visual-metrics: true
+        test: tp6m
+        subtests:
+            - allrecipes
+            - [amazon-search, amazon-s]
+            - espn
+            - facebook
+            - google
+            - [microsoft-support, micros-sup]
+            - [youtube-watch, youtube-w]
     run-on-projects:
         by-app:
             geckoview:
                 by-test-platform:
                     android-hw-g5.*shippable.*: [trunk, mozilla-beta]
                     android-hw-p2.*aarch64-shippable.*: [trunk, mozilla-beta]
                     android-hw-s7-.*: []
                     default: []
@@ -136,110 +142,114 @@ browsertime-tp6m-essential:
                 by-app:
                     geckoview: 1
                     refbrow: 2
                     default: 3
 
 browsertime-youtube-playback-mobile:
     description: Browsertime YouTube Playback on Android
     treeherder-symbol: Btime(ytp)
-    raptor-subtests:
-        by-test-platform:
-            android-hw-g5.*:
-                - [youtube-playback-h264-sfr, ytp-h264-sfr]
-                - [youtube-playback-hfr, ytp-hfr]
-                # Bug 1699469 - Disabled until a crash is fixed.
-                # - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr]
-                # - [youtube-playback-widevine-hfr, ytp-widevine-hfr]
-            android-hw-p2.*:
-                - [youtube-playback-h264-sfr, ytp-h264-sfr]
-                - [youtube-playback-hfr, ytp-hfr]
-                - [youtube-playback-vp9-sfr, ytp-vp9-sfr]
-                # Bug 1699469 - Disabled until a crash is fixed.
-                # - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr]
-                # - [youtube-playback-widevine-hfr, ytp-widevine-hfr]
-                # - [youtube-playback-widevine-vp9-sfr, ytp-widevine-vp9-sfr]
-            android-hw-s7.*: []
-    apps: [fenix, geckoview, refbrow]
+    raptor:
+        apps: [fenix, geckoview, refbrow]
+        subtests:
+            by-test-platform:
+                android-hw-g5.*:
+                    - [youtube-playback-h264-sfr, ytp-h264-sfr]
+                    - [youtube-playback-hfr, ytp-hfr]
+                    # Bug 1699469 - Disabled until a crash is fixed.
+                    # - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr]
+                    # - [youtube-playback-widevine-hfr, ytp-widevine-hfr]
+                android-hw-p2.*:
+                    - [youtube-playback-h264-sfr, ytp-h264-sfr]
+                    - [youtube-playback-hfr, ytp-hfr]
+                    - [youtube-playback-vp9-sfr, ytp-vp9-sfr]
+                    # Bug 1699469 - Disabled until a crash is fixed.
+                    # - [youtube-playback-widevine-h264-sfr, ytp-widevine-h264-sfr]
+                    # - [youtube-playback-widevine-hfr, ytp-widevine-hfr]
+                    # - [youtube-playback-widevine-vp9-sfr, ytp-widevine-vp9-sfr]
+                android-hw-s7.*: []
+        test-url-param:
+            by-subtest:
+                youtube-playback-h264-sfr:  # remove high resolution tests
+                    exclude=1,2,
+                    23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38
+                youtube-playback-hfr:
+                    by-test-platform:
+                        android-hw-g5.*:  # remove VP9(1-34) and AV1(51-74) tests
+                            exclude=1,2,
+                            3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,
+                            51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74
+                        android-hw-p2.*:  # remove AV1(51-74) tests
+                            exclude=1,2,
+                            51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74
+                        default: ""
+                youtube-playback-widevine-hfr:
+                    by-test-platform:
+                        android-hw-g5.*:  # remove VP9(1-34) tests
+                            exclude=1,2,
+                            3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34
+                        default: ""
+                default: ""
     run-on-projects:
         by-app:
             geckoview:
                 by-test-platform:
                     android-hw-g5.*shippable.*: [mozilla-central]
                     android-hw-p2.*aarch64-shippable.*: [mozilla-central]
                     default: []
             default: []
-    test-url-param:
-        by-subtest:
-            youtube-playback-h264-sfr:  # remove high resolution tests
-                exclude=1,2,
-                23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38
-            youtube-playback-hfr:
-                by-test-platform:
-                    android-hw-g5.*:  # remove VP9(1-34) and AV1(51-74) tests
-                        exclude=1,2,
-                        3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,
-                        51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74
-                    android-hw-p2.*:  # remove AV1(51-74) tests
-                        exclude=1,2,
-                        51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74
-                    default: ""
-            youtube-playback-widevine-hfr:
-                by-test-platform:
-                    android-hw-g5.*:  # remove VP9(1-34) tests
-                        exclude=1,2,
-                        3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34
-                    default: ""
-            default: ""
     max-run-time: 3600
 
 browsertime-tp6m-live:
     <<: *tp6m-defaults
-    apps: [chrome-m, fenix, geckoview]
-    raptor-subtests:
-        - allrecipes
-        - amazon
-        - [amazon-search, amazon-s]
-        - bing
-        - [bing-search-restaurants, bing-s-r]
-        - booking
-        - booking-sf
-        - cnn
-        - [cnn-ampstories, cnn-amp]
-        - dailymail
-        - discord
-        - [ebay-kleinanzeigen, ebay-k]
-        - [ebay-kleinanzeigen-search, ebay-k-s]
-        - espn
-        - expedia
-        - facebook
-        - [facebook-cristiano, fb-cris]
-        - [fashionbeans, fashbea]
-        - google
-        - [google-accounts, gacc]
-        - [google-maps, gmaps]
-        - [google-search-restaurants, gsearch-r]
-        - instagram
-        - imdb
-        - [imdb-firefox, imdb-f]
-        - [medium-article, medium]
-        - [microsoft-support, micros-sup]
-        - nytimes
-        - [people-article, people]
-        - reddit
-        - [reddit-thread, reddit-thr]
-        - rumble-fox
-        - [stackoverflow, stacko]
-        - [stackoverflow-question, stack-q]
-        - [urbandictionary-define, urbict-def]
-        - web-de
-        - [wikia-marvel, wikia-m]
-        - wikipedia
-        - youtube
-        - [youtube-watch, youtube-w]
+    raptor:
+        apps: [chrome-m, fenix, geckoview]
+        run-visual-metrics: true
+        test: tp6m
+        subtests:
+            - allrecipes
+            - amazon
+            - [amazon-search, amazon-s]
+            - bing
+            - [bing-search-restaurants, bing-s-r]
+            - booking
+            - booking-sf
+            - cnn
+            - [cnn-ampstories, cnn-amp]
+            - dailymail
+            - discord
+            - [ebay-kleinanzeigen, ebay-k]
+            - [ebay-kleinanzeigen-search, ebay-k-s]
+            - espn
+            - expedia
+            - facebook
+            - [facebook-cristiano, fb-cris]
+            - [fashionbeans, fashbea]
+            - google
+            - [google-accounts, gacc]
+            - [google-maps, gmaps]
+            - [google-search-restaurants, gsearch-r]
+            - instagram
+            - imdb
+            - [imdb-firefox, imdb-f]
+            - [medium-article, medium]
+            - [microsoft-support, micros-sup]
+            - nytimes
+            - [people-article, people]
+            - reddit
+            - [reddit-thread, reddit-thr]
+            - rumble-fox
+            - [stackoverflow, stacko]
+            - [stackoverflow-question, stack-q]
+            - [urbandictionary-define, urbict-def]
+            - web-de
+            - [wikia-marvel, wikia-m]
+            - wikipedia
+            - youtube
+            - [youtube-watch, youtube-w]
     mozharness:
         extra-options:
             - --live-sites
             - --chimera
     run-on-projects:
         by-subtest:
             cnn-ampstories:
                 by-app:
@@ -250,35 +260,37 @@ browsertime-tp6m-live:
                             default: []
                     default: []
             default: []
     tier: 3
     treeherder-symbol: Btime-live(tp6m)
 
 browsertime-tp6m-profiling:
     description: Raptor (browsertime) tp6 page-load tests on android with Gecko profiling
-    raptor-test: tp6m
-    raptor-subtests: [amazon, youtube]
-    apps: [geckoview]
+    raptor:
+        apps: [geckoview]
+        test: tp6m
+        subtests: [amazon, youtube]
     treeherder-symbol: Btime-Prof(tp6m)
     mozharness:
         script: raptor_script.py
         config:
             - raptor/android_hw_config.py
         extra-options:
             - --chimera
             - --gecko-profile
     run-on-projects:
         by-test-platform:
             android-hw-p2-.*aarch64-shippable-qr/opt: [mozilla-central]
             default: []
 
 browsertime-speedometer-mobile:
     description: Browsertime Speedometer on Android
-    apps: [geckoview, fenix, chrome-m]
+    raptor:
+        apps: [geckoview, fenix, chrome-m]
     tier:
         by-app:
             fenix: 1
             default: 3
     run-on-projects:
         by-app:
             refbrow: []
             geckoview:
@@ -293,17 +305,18 @@ browsertime-speedometer-mobile:
     treeherder-symbol: Btime(sp)
     mozharness:
         extra-options:
             - --browsertime
             - --test=speedometer
 
 browsertime-unity-webgl-mobile:
     description: Browsertime Unity WebGL on Android
-    apps: [geckoview, refbrow, fenix, chrome-m]
+    raptor:
+        apps: [geckoview, refbrow, fenix, chrome-m]
     treeherder-symbol: Btime(ugl)
     run-on-projects:
         by-app:
             refbrow: []
             geckoview:
                 by-test-platform:
                     android-hw-g5.*shippable-qr.*: [trunk, mozilla-beta]
                     android-hw-p2.*aarch64-shippable-qr.*: [trunk, mozilla-beta]
@@ -317,19 +330,20 @@ browsertime-unity-webgl-mobile:
         extra-options:
             - --test=unity-webgl
     fetches:
         fetch:
             - unity-webgl
 
 browsertime-power:
     description: Browsertime Power Usage Tests on Android
-    apps: [geckoview, fenix, refbrow]
+    raptor:
+        apps: [geckoview, fenix, refbrow]
+        subtests:
+            - idle
+            - idle-bg
+            - [speedometer, sp]
     tier: 2
-    raptor-subtests:
-        - idle
-        - idle-bg
-        - [speedometer, sp]
     run-on-projects: []
     treeherder-symbol: Btime-P(power)
     mozharness:
         extra-options:
             - --power-test
--- a/taskcluster/gecko_taskgraph/transforms/test/__init__.py
+++ b/taskcluster/gecko_taskgraph/transforms/test/__init__.py
@@ -472,20 +472,21 @@ test_description_schema = Schema(
                 None,
                 {Required("index"): str, Required("name"): str},
             ),
         ),
         # A list of artifacts to install from 'fetch' tasks.
         Optional("fetches"): {str: optionally_keyed_by("test-platform", [str])},
         # Opt-in to Python 3 support
         Optional("python-3"): bool,
-        # Raptor / browsertime specific keys that need to be here to support
-        # using `by-key` after `by-variant`. Ideally these keys should not exist
-        # in the tests.py schema and instead we'd split variants before the raptor
-        # transforms need them. See bug 1700774.
+        # Raptor / browsertime specific keys, defer validation to 'raptor.py'
+        # transform.
+        Optional("raptor"): object,
+        # Raptor / browsertime specific keys that need to be here since 'raptor' schema
+        # is evluated *before* test_description_schema
         Optional("app"): str,
         Optional("subtest"): str,
         # Define if a given task supports artifact builds or not, see bug 1695325.
         Optional("supports-artifact-builds"): bool,
     }
 )
 
 
--- a/taskcluster/gecko_taskgraph/transforms/test/raptor.py
+++ b/taskcluster/gecko_taskgraph/transforms/test/raptor.py
@@ -16,44 +16,46 @@ from gecko_taskgraph.transforms.test imp
 from gecko_taskgraph.util.schema import optionally_keyed_by, resolve_keyed_by, Schema
 from gecko_taskgraph.util.treeherder import split_symbol, join_symbol
 
 transforms = TransformSequence()
 
 raptor_description_schema = Schema(
     {
         # Raptor specific configs.
-        Optional("apps"): optionally_keyed_by("test-platform", "subtest", [str]),
-        Optional("raptor-test"): str,
-        Optional("raptor-subtests"): optionally_keyed_by("app", "test-platform", list),
-        Optional("activity"): optionally_keyed_by("app", str),
-        Optional("binary-path"): optionally_keyed_by("app", str),
+        Optional("raptor"): {
+            Optional("activity"): optionally_keyed_by("app", str),
+            Optional("apps"): optionally_keyed_by("test-platform", "subtest", [str]),
+            Optional("binary-path"): optionally_keyed_by("app", str),
+            Optional("run-visual-metrics"): optionally_keyed_by("app", bool),
+            Optional("subtests"): optionally_keyed_by("app", "test-platform", list),
+            Optional("test"): str,
+            Optional("test-url-param"): optionally_keyed_by(
+                "subtest", "test-platform", str
+            ),
+        },
         # Configs defined in the 'test_description_schema'.
         Optional("max-run-time"): optionally_keyed_by(
             "app", "subtest", "test-platform", test_description_schema["max-run-time"]
         ),
         Optional("run-on-projects"): optionally_keyed_by(
             "app",
             "test-name",
-            "raptor-test",
+            "raptor.test",
             "subtest",
             "variant",
             test_description_schema["run-on-projects"],
         ),
         Optional("variants"): test_description_schema["variants"],
         Optional("target"): optionally_keyed_by(
             "app", test_description_schema["target"]
         ),
         Optional("tier"): optionally_keyed_by(
-            "app", "raptor-test", "subtest", "variant", test_description_schema["tier"]
+            "app", "raptor.test", "subtest", "variant", test_description_schema["tier"]
         ),
-        Optional("test-url-param"): optionally_keyed_by(
-            "subtest", "test-platform", str
-        ),
-        Optional("run-visual-metrics"): optionally_keyed_by("app", bool),
         Required("test-name"): test_description_schema["test-name"],
         Required("test-platform"): test_description_schema["test-platform"],
         Required("require-signed-extensions"): test_description_schema[
             "require-signed-extensions"
         ],
         Required("treeherder-symbol"): test_description_schema["treeherder-symbol"],
         # Any unrecognized keys will be validated against the test_description_schema.
         Extra: object,
@@ -61,32 +63,32 @@ raptor_description_schema = Schema(
 )
 
 transforms.add_validate(raptor_description_schema)
 
 
 @transforms.add
 def set_defaults(config, tests):
     for test in tests:
-        test.setdefault("run-visual-metrics", False)
+        test.setdefault("raptor", {}).setdefault("run-visual-metrics", False)
         yield test
 
 
 @transforms.add
 def split_apps(config, tests):
     app_symbols = {
         "chrome": "ChR",
         "chrome-m": "ChR",
         "chromium": "Cr",
         "fenix": "fenix",
         "refbrow": "refbrow",
     }
 
     for test in tests:
-        apps = test.pop("apps", None)
+        apps = test["raptor"].pop("apps", None)
         if not apps:
             yield test
             continue
 
         for app in apps:
             # Ignore variants for non-Firefox applications.
             if app != "firefox" and test["attributes"].get("unittest_variant"):
                 continue
@@ -111,26 +113,26 @@ def split_apps(config, tests):
 @transforms.add
 def handle_keyed_by_prereqs(config, tests):
     """
     Only resolve keys for prerequisite fields here since the
     these keyed-by options might have keyed-by fields
     as well.
     """
     for test in tests:
-        resolve_keyed_by(test, "raptor-subtests", item_name=test["test-name"])
+        resolve_keyed_by(test, "raptor.subtests", item_name=test["test-name"])
         yield test
 
 
 @transforms.add
 def split_raptor_subtests(config, tests):
     for test in tests:
-        # For tests that have 'raptor-subtests' listed, we want to create a separate
+        # For tests that have 'subtests' listed, we want to create a separate
         # test job for every subtest (i.e. split out each page-load URL into its own job)
-        subtests = test.pop("raptor-subtests", None)
+        subtests = test["raptor"].pop("subtests", None)
         if not subtests:
             yield test
             continue
 
         chunk_number = 0
 
         for subtest in subtests:
             chunk_number += 1
@@ -147,40 +149,40 @@ def split_raptor_subtests(config, tests)
                 chunked, "tier", chunked["subtest"], defer=["variant"]
             )
             yield chunked
 
 
 @transforms.add
 def handle_keyed_by(config, tests):
     fields = [
-        "test-url-param",
+        "raptor.test-url-param",
+        "raptor.run-visual-metrics",
+        "raptor.activity",
+        "raptor.binary-path",
         "limit-platforms",
-        "activity",
-        "binary-path",
         "fetches.fetch",
         "max-run-time",
         "run-on-projects",
         "target",
         "tier",
-        "run-visual-metrics",
     ]
     for test in tests:
         for field in fields:
             resolve_keyed_by(
                 test, field, item_name=test["test-name"], defer=["variant"]
             )
         yield test
 
 
 @transforms.add
 def split_page_load_by_url(config, tests):
     for test in tests:
         # `chunk-number` and 'subtest' only exists when the task had a
-        # definition for `raptor-subtests`
+        # definition for `subtests`
         chunk_number = test.pop("chunk-number", None)
         subtest = test.get(
             "subtest"
         )  # don't pop as some tasks need this value after splitting variants
         subtest_symbol = test.pop("subtest-symbol", None)
 
         if not chunk_number or not subtest:
             yield test
@@ -188,23 +190,23 @@ def split_page_load_by_url(config, tests
 
         if len(subtest_symbol) > 10 and "ytp" not in subtest_symbol:
             raise Exception(
                 "Treeherder symbol %s is lager than 10 char! Please use a different symbol."
                 % subtest_symbol
             )
 
         if test["test-name"].startswith("browsertime-"):
-            test["raptor-test"] = subtest
+            test["raptor"]["test"] = subtest
 
             # Remove youtube-playback in the test name to avoid duplication
             test["test-name"] = test["test-name"].replace("youtube-playback-", "")
         else:
             # Use full test name if running on webextension
-            test["raptor-test"] = "raptor-tp6-" + subtest + "-{}".format(test["app"])
+            test["raptor"]["test"] = "raptor-tp6-" + subtest + "-{}".format(test["app"])
 
         # Only run the subtest/single URL
         test["test-name"] += f"-{subtest}"
         test["try-name"] += f"-{subtest}"
 
         # Set treeherder symbol and description
         group, _ = split_symbol(test["treeherder-symbol"])
         test["treeherder-symbol"] = join_symbol(group, subtest_symbol)
@@ -257,39 +259,41 @@ def add_extra_options(config, tests):
 
         # Adding device name if we're on android
         test_platform = test["test-platform"]
         if test_platform.startswith("android-hw-g5"):
             extra_options.append("--device-name=g5")
         elif test_platform.startswith("android-hw-p2"):
             extra_options.append("--device-name=p2_aarch64")
 
-        if test.pop("run-visual-metrics", False):
+        if test["raptor"].pop("run-visual-metrics", False):
             extra_options.append("--browsertime-video")
             test["attributes"]["run-visual-metrics"] = True
 
         if "app" in test:
             extra_options.append(
                 "--app={}".format(test["app"])
             )  # don't pop as some tasks need this value after splitting variants
 
-        if "activity" in test:
-            extra_options.append("--activity={}".format(test.pop("activity")))
+        if "activity" in test["raptor"]:
+            extra_options.append("--activity={}".format(test["raptor"].pop("activity")))
 
-        if "binary-path" in test:
-            extra_options.append("--binary-path={}".format(test.pop("binary-path")))
+        if "binary-path" in test["raptor"]:
+            extra_options.append(
+                "--binary-path={}".format(test["raptor"].pop("binary-path"))
+            )
 
-        if "raptor-test" in test:
-            extra_options.append("--test={}".format(test.pop("raptor-test")))
+        if "test" in test["raptor"]:
+            extra_options.append("--test={}".format(test["raptor"].pop("test")))
 
         if test["require-signed-extensions"]:
             extra_options.append("--is-release-build")
 
-        if "test-url-param" in test:
-            param = test.pop("test-url-param")
+        if "test-url-param" in test["raptor"]:
+            param = test["raptor"].pop("test-url-param")
             if not param == []:
                 extra_options.append(
                     "--test-url-params={}".format(param.replace(" ", ""))
                 )
 
         extra_options.append("--project={}".format(config.params.get("project")))
 
         yield test