Bug 1580722 - Create cold tests for all mobile websites r=perftest-reviewers,stephendonner
authorMarian Raiciof <mraiciof@mozilla.com>
Mon, 16 Sep 2019 18:47:46 +0000
changeset 493773 45b3bf9cfeccd8b057c95440be77b7ae61428de0
parent 493772 d7bfa93013b461dbe44eef290d22b3c839e33934
child 493774 8a07bd671d5337bdcaea69d220cf2a27cf6876eb
push id36589
push usernerli@mozilla.com
push dateWed, 18 Sep 2019 21:49:27 +0000
treeherdermozilla-central@21aff209f5a9 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest-reviewers, stephendonner
bugs1580722
milestone71.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1580722 - Create cold tests for all mobile websites r=perftest-reviewers,stephendonner This task has been added to create all the missing cold tests for mobile websites. Differential Revision: https://phabricator.services.mozilla.com/D45634
taskcluster/ci/test/raptor-fenix.yml
taskcluster/ci/test/raptor-fennec.yml
taskcluster/ci/test/raptor-gve.yml
taskcluster/ci/test/raptor-refbrow.yml
taskcluster/ci/test/test-sets.yml
testing/raptor/raptor/raptor.ini
testing/raptor/raptor/tests/tp6/mobile/raptor-tp6m-3.ini
testing/raptor/raptor/tests/tp6/mobile/raptor-tp6m-cold-28.ini
--- a/taskcluster/ci/test/raptor-fenix.yml
+++ b/taskcluster/ci/test/raptor-fenix.yml
@@ -541,16 +541,30 @@ raptor-tp6m-27-fenix-cold:
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-tp6m-cold-27
             - --app=fenix
             - --binary-path=org.mozilla.fenix.performancetest
             - --activity=org.mozilla.fenix.IntentReceiverActivity
 
+raptor-tp6m-28-fenix-cold:
+    description: "Raptor tp6m-28 cold page-load on Fenix"
+    try-name: raptor-tp6m-28-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-28)
+    run-on-projects: []
+    e10s: true
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-28
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix.performancetest
+            - --activity=org.mozilla.fenix.IntentReceiverActivity
+
 raptor-scn-power-idle-fenix:
     description: "Raptor idle-browser power-usage test on Fenix"
     try-name: raptor-scn-power-idle-fenix
     treeherder-symbol: Rap-fenix(idl-p)
     run-on-projects: []
     e10s: true
     tier: 3
     max-run-time: 1800
--- a/taskcluster/ci/test/raptor-fennec.yml
+++ b/taskcluster/ci/test/raptor-fennec.yml
@@ -597,8 +597,21 @@ raptor-tp6m-27-fennec-cold:
     treeherder-symbol: Rap(tp6m-c-27)
     run-on-projects: []
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-tp6m-cold-27
             - --app=fennec
             - --binary=org.mozilla.fennec_aurora
+
+raptor-tp6m-28-fennec-cold:
+    description: "Raptor tp6m-28 cold page-load on Fennec"
+    max-run-time: 2700
+    try-name: raptor-tp6m-28-fennec-cold
+    treeherder-symbol: Rap(tp6m-c-28)
+    run-on-projects: []
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-28
+            - --app=fennec
+            - --binary=org.mozilla.fennec_aurora
--- a/taskcluster/ci/test/raptor-gve.yml
+++ b/taskcluster/ci/test/raptor-gve.yml
@@ -896,8 +896,28 @@ raptor-tp6m-27-geckoview-cold:
             default: ['trunk', 'mozilla-beta']
     e10s: true
     mozharness:
         extra-options:
             - --test=raptor-tp6m-cold-27
             - --app=geckoview
             - --binary=org.mozilla.geckoview_example
             - --activity=org.mozilla.geckoview_example.GeckoViewActivity
+
+raptor-tp6m-28-geckoview-cold:
+    description: "Raptor tp6m-28 cold page-load on GeckoView Example"
+    max-run-time: 2700
+    try-name: raptor-tp6m-28-geckoview-cold
+    treeherder-symbol: Rap(tp6m-c-28)
+    run-on-projects:
+        by-test-platform:
+            android-hw-.*/opt: ['mozilla-central']
+            android-hw-p2-.*-api-16/pgo: ['trunk']
+            android-hw-p2-.*aarch64.*/pgo: ['trunk', 'mozilla-beta']
+            android-hw-g5.*/pgo: ['mozilla-central']
+            default: ['trunk', 'mozilla-beta']
+    e10s: true
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-28
+            - --app=geckoview
+            - --binary=org.mozilla.geckoview_example
+            - --activity=org.mozilla.geckoview_example.GeckoViewActivity
--- a/taskcluster/ci/test/raptor-refbrow.yml
+++ b/taskcluster/ci/test/raptor-refbrow.yml
@@ -558,8 +558,20 @@ raptor-tp6m-27-refbrow-cold:
     treeherder-symbol: Rap-refbrow(tp6m-c-27)
     e10s: true
     mozharness:
         extra-options:
             - --test=raptor-tp6m-cold-27
             - --app=refbrow
             - --binary-path=org.mozilla.reference.browser.raptor
             - --activity=org.mozilla.reference.browser.BrowserTestActivity
+
+raptor-tp6m-28-refbrow-cold:
+    description: "Raptor tp6m-28 cold page-load on the reference browser"
+    try-name: raptor-tp6m-28-refbrow-cold
+    treeherder-symbol: Rap-refbrow(tp6m-c-28)
+    e10s: true
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-28
+            - --app=refbrow
+            - --binary-path=org.mozilla.reference.browser.raptor
+            - --activity=org.mozilla.reference.browser.BrowserTestActivity
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -496,16 +496,17 @@ android-hw-arm7-raptor:
     - raptor-tp6m-20-fenix-cold
     - raptor-tp6m-21-fenix-cold
     - raptor-tp6m-22-fenix-cold
     - raptor-tp6m-23-fenix-cold
     - raptor-tp6m-24-fenix-cold
     - raptor-tp6m-25-fenix-cold
     - raptor-tp6m-26-fenix-cold
     - raptor-tp6m-27-fenix-cold
+    - raptor-tp6m-28-fenix-cold
     - raptor-tp6m-1-geckoview-cold
     - raptor-tp6m-2-geckoview-cold
     - raptor-tp6m-3-geckoview-cold
     - raptor-tp6m-4-geckoview-cold
     - raptor-tp6m-5-geckoview-cold
     - raptor-tp6m-6-geckoview-cold
     - raptor-tp6m-7-geckoview-cold
     - raptor-tp6m-8-geckoview-cold
@@ -523,16 +524,17 @@ android-hw-arm7-raptor:
     - raptor-tp6m-20-geckoview-cold
     - raptor-tp6m-21-geckoview-cold
     - raptor-tp6m-22-geckoview-cold
     - raptor-tp6m-23-geckoview-cold
     - raptor-tp6m-24-geckoview-cold
     - raptor-tp6m-25-geckoview-cold
     - raptor-tp6m-26-geckoview-cold
     - raptor-tp6m-27-geckoview-cold
+    - raptor-tp6m-28-geckoview-cold
     - raptor-tp6m-1-fennec64
     - raptor-tp6m-2-fennec64
     - raptor-tp6m-3-fennec64
     - raptor-tp6m-4-fennec64
     - raptor-tp6m-5-fennec64
     - raptor-tp6m-6-fennec64
     - raptor-tp6m-7-fennec64
     - raptor-tp6m-8-fennec64
@@ -570,16 +572,17 @@ android-hw-arm7-raptor:
     - raptor-tp6m-20-fennec-cold
     - raptor-tp6m-21-fennec-cold
     - raptor-tp6m-22-fennec-cold
     - raptor-tp6m-23-fennec-cold
     - raptor-tp6m-24-fennec-cold
     - raptor-tp6m-25-fennec-cold
     - raptor-tp6m-26-fennec-cold
     - raptor-tp6m-27-fennec-cold
+    - raptor-tp6m-28-fennec-cold
     - raptor-tp6m-1-fennec64-cold
     - raptor-tp6m-2-fennec64-cold
     - raptor-tp6m-3-fennec64-cold
     - raptor-tp6m-4-fennec64-cold
     - raptor-tp6m-5-fennec64-cold
     - raptor-tp6m-6-fennec64-cold
     - raptor-tp6m-7-fennec64-cold
     - raptor-tp6m-8-fennec64-cold
@@ -625,16 +628,17 @@ android-hw-arm7-raptor:
     - raptor-tp6m-20-refbrow-cold
     - raptor-tp6m-21-refbrow-cold
     - raptor-tp6m-22-refbrow-cold
     - raptor-tp6m-23-refbrow-cold
     - raptor-tp6m-24-refbrow-cold
     - raptor-tp6m-25-refbrow-cold
     - raptor-tp6m-26-refbrow-cold
     - raptor-tp6m-27-refbrow-cold
+    - raptor-tp6m-28-refbrow-cold
 
 android-hw-aarch64-raptor:
     - raptor-speedometer-geckoview
     - raptor-speedometer-refbrow
     - raptor-youtube-playback-fenix
     - raptor-youtube-playback-geckoview
     - raptor-tp6m-1-geckoview
     - raptor-tp6m-2-geckoview
@@ -688,16 +692,17 @@ android-hw-aarch64-raptor:
     - raptor-tp6m-20-fenix-cold
     - raptor-tp6m-21-fenix-cold
     - raptor-tp6m-22-fenix-cold
     - raptor-tp6m-23-fenix-cold
     - raptor-tp6m-24-fenix-cold
     - raptor-tp6m-25-fenix-cold
     - raptor-tp6m-26-fenix-cold
     - raptor-tp6m-27-fenix-cold
+    - raptor-tp6m-28-fenix-cold
     - raptor-tp6m-1-geckoview-cold
     - raptor-tp6m-2-geckoview-cold
     - raptor-tp6m-3-geckoview-cold
     - raptor-tp6m-4-geckoview-cold
     - raptor-tp6m-5-geckoview-cold
     - raptor-tp6m-6-geckoview-cold
     - raptor-tp6m-7-geckoview-cold
     - raptor-tp6m-8-geckoview-cold
@@ -715,16 +720,17 @@ android-hw-aarch64-raptor:
     - raptor-tp6m-20-geckoview-cold
     - raptor-tp6m-21-geckoview-cold
     - raptor-tp6m-22-geckoview-cold
     - raptor-tp6m-23-geckoview-cold
     - raptor-tp6m-24-geckoview-cold
     - raptor-tp6m-25-geckoview-cold
     - raptor-tp6m-26-geckoview-cold
     - raptor-tp6m-27-geckoview-cold
+    - raptor-tp6m-28-geckoview-cold
     - raptor-tp6m-1-fennec-cold
     - raptor-tp6m-2-fennec-cold
     - raptor-tp6m-3-fennec-cold
     - raptor-tp6m-4-fennec-cold
     - raptor-tp6m-5-fennec-cold
     - raptor-tp6m-6-fennec-cold
     - raptor-tp6m-7-fennec-cold
     - raptor-tp6m-8-fennec-cold
@@ -742,16 +748,17 @@ android-hw-aarch64-raptor:
     - raptor-tp6m-20-fennec-cold
     - raptor-tp6m-21-fennec-cold
     - raptor-tp6m-22-fennec-cold
     - raptor-tp6m-23-fennec-cold
     - raptor-tp6m-24-fennec-cold
     - raptor-tp6m-25-fennec-cold
     - raptor-tp6m-26-fennec-cold
     - raptor-tp6m-27-fennec-cold
+    - raptor-tp6m-28-fennec-cold
     - raptor-tp6m-1-fennec64-cold
     - raptor-tp6m-2-fennec64-cold
     - raptor-tp6m-3-fennec64-cold
     - raptor-tp6m-4-fennec64-cold
     - raptor-tp6m-5-fennec64-cold
     - raptor-tp6m-6-fennec64-cold
     - raptor-tp6m-7-fennec64-cold
     - raptor-tp6m-8-fennec64-cold
@@ -797,16 +804,17 @@ android-hw-aarch64-raptor:
     - raptor-tp6m-20-refbrow-cold
     - raptor-tp6m-21-refbrow-cold
     - raptor-tp6m-22-refbrow-cold
     - raptor-tp6m-23-refbrow-cold
     - raptor-tp6m-24-refbrow-cold
     - raptor-tp6m-25-refbrow-cold
     - raptor-tp6m-26-refbrow-cold
     - raptor-tp6m-27-refbrow-cold
+    - raptor-tp6m-28-refbrow-cold
 
 android-hw-arm7-raptor-power:
     - raptor-speedometer-geckoview-power
     - raptor-scn-power-idle-fenix
     - raptor-scn-power-idle-fennec
     - raptor-scn-power-idle-geckoview
     - raptor-scn-power-idle-refbrow
     - raptor-scn-power-idle-bg-fenix
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@@ -74,16 +74,17 @@
 [include:tests/tp6/mobile/raptor-tp6m-cold-20.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-21.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-22.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-23.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-24.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-25.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-26.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-27.ini]
+[include:tests/tp6/mobile/raptor-tp6m-cold-28.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-1-fennec64.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-2-fennec64.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-3-fennec64.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-4-fennec64.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-5-fennec64.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-6-fennec64.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-7-fennec64.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-8-fennec64.ini]
--- a/testing/raptor/raptor/tests/tp6/mobile/raptor-tp6m-3.ini
+++ b/testing/raptor/raptor/tests/tp6/mobile/raptor-tp6m-3.ini
@@ -36,17 +36,16 @@ playback_recordings = mitm4-pixel2-fenne
 measure = fnbpaint, dcf, loadtime
 alert_on = loadtime
 
 [raptor-tp6m-google-restaurants-fennec]
 apps = fennec
 test_url = https://www.google.com/search?q=restaurants+near+me
 playback_pageset_manifest = mitm4-pixel2-fennec-google-search-restaurants.manifest
 playback_recordings = mitm4-pixel2-fennec-google-search-restaurants.mp
-
 measure = fnbpaint, dcf, loadtime
 alert_on = loadtime
 
 [raptor-tp6m-google-maps-refbrow]
 apps = refbrow
 test_url = https://www.google.com/maps?force=pwa
 playback_pageset_manifest = mitm4-pixel2-fennec-google-maps.manifest
 playback_recordings = mitm4-pixel2-fennec-google-maps.mp
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/tp6/mobile/raptor-tp6m-cold-28.ini
@@ -0,0 +1,36 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-28
+
+[DEFAULT]
+type =  pageload
+test_url = https://www.google.com/search?q=restaurants+near+me
+playback = mitmproxy-android
+playback_pageset_manifest = mitm4-pixel2-fennec-google-search-restaurants.manifest
+playback_recordings = mitm4-pixel2-fennec-google-search-restaurants.mp
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-google-restaurants-fenix-cold]
+apps = fenix
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-google-restaurants-geckoview-cold]
+apps = geckoview
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-google-restaurants-fennec-cold]
+apps = fennec
+measure = fnbpaint, dcf, loadtime
+alert_on = loadtime
+
+[raptor-tp6m-google-restaurants-refbrow-cold]
+apps = refbrow
+measure = fnbpaint, fcp, dcf, loadtime