Bug 1615255 - Enable all geckoview browsertime pageload tests. r=perftest-reviewers,Bebe,jmaher
authorGregory Mierzwinski <gmierz2@outlook.com>
Wed, 20 May 2020 16:53:07 +0000
changeset 531287 0c95f4b8b39acb1fd1f579c5356c062e689cd42e
parent 531286 bdaa296c4e6647e0e0e650a53e91c40cef1c13ff
child 531288 354ce68d14f8f9499d05c5912b4b7e886e87ff4e
push id37436
push userncsoregi@mozilla.com
push dateWed, 20 May 2020 21:30:50 +0000
treeherdermozilla-central@6c10970490f3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest-reviewers, Bebe, jmaher
bugs1615255
milestone78.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1615255 - Enable all geckoview browsertime pageload tests. r=perftest-reviewers,Bebe,jmaher This patch enables all cold and warm pageload tests for browsertime anywhere that the raptor tests are running. Differential Revision: https://phabricator.services.mozilla.com/D74986
taskcluster/ci/test/browsertime-mobile.yml
taskcluster/ci/visual-metrics-dep/kind.yml
taskcluster/taskgraph/transforms/raptor.py
--- a/taskcluster/ci/test/browsertime-mobile.yml
+++ b/taskcluster/ci/test/browsertime-mobile.yml
@@ -58,23 +58,32 @@ job-defaults:
             refbrow: org.mozilla.reference.browser.raptor
             default: None
     mozharness:
         script: raptor_script.py
         config:
             - raptor/android_hw_config.py
         extra-options:
             - --browsertime
+    optimization:
+        push-interval-25:
+            - android
 
 browsertime-tp6m:
     description: "Raptor (browsertime) tp6 page-load tests on android"
     raptor-test: tp6m
     raptor-subtests:
         by-app:
-            fenix:
+            fennec:
+                - amazon
+                - youtube
+            chrome-m:
+                - amazon
+                - youtube
+            default:
                 - allrecipes
                 - amazon
                 - amazon-search
                 - bbc
                 - bing
                 - bing-search-restaurants
                 - booking
                 - cnn
@@ -92,49 +101,95 @@ browsertime-tp6m:
                 - jianshu
                 - microsoft-support
                 - reddit
                 - stackoverflow
                 - web-de
                 - wikipedia
                 - youtube
                 - youtube-watch
-            default:
-                - amazon
-                - youtube
     apps: ["geckoview", "fenix", "fennec", "chrome-m"]
     tier:
         by-raptor-test:
             amazon:
                 by-app:
                     fenix: 1
+                    geckoview: 1
                     default: 3
-            default: 3
+            default:
+                by-app:
+                    geckoview: 1
+                    default: 3
     run-on-projects:
         by-raptor-test:
             amazon:
                 by-app:
                     fenix: []
                     chrome-m: []
+                    geckoview:
+                        by-pageload:
+                            cold:
+                                by-test-platform:
+                                    android-hw-.*/opt: []
+                                    android-hw-p2-.*api-16/pgo: []
+                                    default: ['trunk', 'mozilla-beta']
+                            warm:
+                                by-test-platform:
+                                    android-hw-p2-.*-api-16/(?!opt).*: []
+                                    android-hw-p2-.*(?!-api-16)/opt.*: []
+                                    android-hw(?!-p2).*(?!-api-16)/opt.*: []
+                                    default: ['mozilla-central']
                     default:
                         by-test-platform:
                             android-hw-p2-.*aarch64.*/pgo: ["mozilla-central", "try"]
                             default: []
             youtube:
                 by-app:
                     fenix: []
                     chrome-m: []
+                    geckoview:
+                        by-pageload:
+                            cold:
+                                by-test-platform:
+                                    android-hw-.*/opt: []
+                                    android-hw-p2-.*api-16/pgo: []
+                                    default: ['trunk', 'mozilla-beta']
+                            warm:
+                                by-test-platform:
+                                    android-hw-p2-.*-api-16/(?!opt).*: []
+                                    android-hw-p2-.*(?!-api-16)/opt.*: []
+                                    android-hw(?!-p2).*(?!-api-16)/opt.*: []
+                                    default: ['mozilla-central']
                     default:
                         by-test-platform:
                             android-hw-p2-.*aarch64.*/pgo: ["mozilla-central", "try"]
                             default: []
-            default: []
+            default:
+                by-pageload:
+                    cold:
+                        by-app:
+                            geckoview:
+                                by-test-platform:
+                                    android-hw-.*/opt: []
+                                    android-hw-p2-.*api-16/pgo: []
+                                    default: ['trunk', 'mozilla-beta']
+                            default: []
+                    warm:
+                        by-app:
+                            geckoview:
+                                by-test-platform:
+                                    android-hw-p2-.*-api-16/(?!opt).*: []
+                                    android-hw-p2-.*(?!-api-16)/opt.*: []
+                                    android-hw(?!-p2).*(?!-api-16)/opt.*: []
+                                    default: ['mozilla-central']
+                            default: []
     pageload:
         by-app:
             fenix: both
+            geckoview: both
             default: cold
     run-visual-metrics:
         by-app:
             fennec: false
             default: true
     treeherder-symbol: Btime(tp6m)
 
 browsertime-tp6m-live:
--- a/taskcluster/ci/visual-metrics-dep/kind.yml
+++ b/taskcluster/ci/visual-metrics-dep/kind.yml
@@ -36,8 +36,11 @@ job-template:
               path: /builds/worker/artifacts/
     fetches:
         fetch:
             - visual-metrics
     run:
         using: run-task
         command: /builds/worker/bin/run-visual-metrics.py -- --orange --perceptual --contentful --force --renderignore 5 --json --viewport
         checkout: false
+    optimization:
+        push-interval-25:
+            - android
--- a/taskcluster/taskgraph/transforms/raptor.py
+++ b/taskcluster/taskgraph/transforms/raptor.py
@@ -48,16 +48,17 @@ raptor_description_schema = Schema({
     ),
     # Configs defined in the 'test_description_schema'.
     Optional('max-run-time'): optionally_keyed_by(
         'app',
         test_description_schema['max-run-time']
     ),
     Optional('run-on-projects'): optionally_keyed_by(
         'app',
+        'pageload',
         'test-name',
         'raptor-test',
         test_description_schema['run-on-projects']
     ),
     Optional('variants'): optionally_keyed_by(
         'app',
         test_description_schema['variants']
     ),
@@ -128,26 +129,37 @@ def split_apps(config, tests):
                 group, symbol = split_symbol(atest['treeherder-symbol'])
                 group += "-{}".format(app_symbols[app])
                 atest['treeherder-symbol'] = join_symbol(group, symbol)
 
             yield atest
 
 
 @transforms.add
-def handle_keyed_by_app(config, tests):
+def handle_keyed_by_prereqs(config, tests):
     """
-    Only resolve keys for raptor-subtests here since the
-    `raptor-test` keyed-by option might have keyed-by fields
+    Only resolve keys for prerequisite fields here since the
+    these keyed-by options might have keyed-by fields
     as well.
     """
-    fields = ['raptor-subtests']
+    fields = ['raptor-subtests', 'pageload']
     for test in tests:
         for field in fields:
             resolve_keyed_by(test, field, item_name=test['test-name'])
+
+        # We need to make the split immediately so that we can split
+        # task configurations by pageload type, the `both` condition is
+        # the same as not having a by-pageload split.
+        if test['pageload'] == 'both':
+            test['pageload'] = 'cold'
+
+            warmtest = deepcopy(test)
+            warmtest['pageload'] = 'warm'
+            yield warmtest
+
         yield test
 
 
 @transforms.add
 def split_raptor_subtests(config, tests):
     for test in tests:
         # for tests that have 'raptor-subtests' listed, we want to create a separate
         # test job for every subtest (i.e. split out each page-load URL into its own job)
@@ -171,17 +183,16 @@ def split_raptor_subtests(config, tests)
 
 @transforms.add
 def handle_keyed_by(config, tests):
     fields = [
         'variants',
         'limit-platforms',
         'activity',
         'binary-path',
-        'pageload',
         'max-run-time',
         'run-on-projects',
         'target',
         'tier',
         'run-visual-metrics'
     ]
     for test in tests:
         for field in fields: