Bug 1567177 - Create cold tests for cnn mobile website. r=perftest-reviewers,sparky
authorMarian Raiciof <mraiciof@mozilla.com>
Wed, 24 Jul 2019 16:08:06 +0000
changeset 484660 5805cd9ae2947386263069e1a3b2d832384bd45f
parent 484659 ee1ac72b98fb91e59396591aa8e02d931454587c
child 484661 c314c3bab4f2dbdc3a0503d2f35f2c07b5a9370f
child 484690 8de4e4796a2f8927707a668b92ccabdc6f0ced93
push id36341
push useraciure@mozilla.com
push dateThu, 25 Jul 2019 09:55:18 +0000
treeherdermozilla-central@5805cd9ae294 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest-reviewers, sparky
bugs1567177
milestone70.0a1
first release with
nightly linux32
5805cd9ae294 / 70.0a1 / 20190725095518 / files
nightly linux64
5805cd9ae294 / 70.0a1 / 20190725095518 / files
nightly mac
5805cd9ae294 / 70.0a1 / 20190725095518 / files
nightly win32
5805cd9ae294 / 70.0a1 / 20190725095518 / files
nightly win64
5805cd9ae294 / 70.0a1 / 20190725095518 / files
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
releases
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1567177 - Create cold tests for cnn mobile website. r=perftest-reviewers,sparky The new cold cnn mobile tests can be found in 'raptor-tp6m-cold-27.ini'. Differential Revision: https://phabricator.services.mozilla.com/D38671
taskcluster/ci/test/raptor.yml
taskcluster/ci/test/test-sets.yml
testing/raptor/raptor/raptor.ini
testing/raptor/raptor/tests/tp6/mobile/raptor-tp6m-cold-27.ini
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -1531,16 +1531,38 @@ raptor-tp6m-26-fenix-cold:
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-tp6m-cold-26
             - --app=fenix
             - --binary-path=org.mozilla.fenix.performancetest
             - --activity=org.mozilla.fenix.IntentReceiverActivity
 
+raptor-tp6m-27-fenix-cold:
+    description: "Raptor tp6m-27 cold page-load on Fenix"
+    try-name: raptor-tp6m-27-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-27)
+    run-on-projects: []
+    e10s: true
+    target:
+        by-test-platform:
+            android-hw.*-aarch64.*/.*:
+                index: project.mobile.fenix.v2.performance-test.latest.aarch64
+                name: target.apk
+            default:
+                index: project.mobile.fenix.v2.performance-test.latest.arm
+                name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-27
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix.performancetest
+            - --activity=org.mozilla.fenix.IntentReceiverActivity
+
 raptor-speedometer-firefox:
     description: "Raptor Speedometer on Firefox"
     variants: ["fission"]
     try-name: raptor-speedometer-firefox
     treeherder-symbol: Rap(sp)
     mozharness:
         extra-options:
             - --test=raptor-speedometer
@@ -2883,16 +2905,38 @@ raptor-tp6m-26-geckoview-cold:
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-tp6m-cold-26
             - --app=geckoview
             - --binary=org.mozilla.geckoview_example
             - --activity=org.mozilla.geckoview_example.GeckoViewActivity
 
+raptor-tp6m-27-geckoview-cold:
+    description: "Raptor tp6m-27 cold page-load on GeckoView Example"
+    max-run-time: 2700
+    try-name: raptor-tp6m-27-geckoview-cold
+    treeherder-symbol: Rap(tp6m-c-27)
+    run-on-projects:
+        by-test-platform:
+            android-hw-.*/opt: ['mozilla-central', 'comm-central']
+            android-hw-p2-.*-api-16/pgo: ['trunk']
+            android-hw-p2-.*aarch64.*/pgo: ['trunk', 'mozilla-beta']
+            android-hw-g5.*/pgo: ['trunk', 'mozilla-beta']
+            default: ['trunk', 'mozilla-beta']
+    e10s: true
+    target: geckoview_example.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-27
+            - --app=geckoview
+            - --binary=org.mozilla.geckoview_example
+            - --activity=org.mozilla.geckoview_example.GeckoViewActivity
+
 raptor-tp6m-1-fennec-cold:
     description: "Raptor tp6m-1 cold page-load on Fennec"
     max-run-time: 2700
     try-name: raptor-tp6m-1-fennec-cold
     treeherder-symbol: Rap(tp6m-c-1)
     target: target.apk
     run-on-projects: []
     tier: 2
@@ -3247,16 +3291,30 @@ raptor-tp6m-26-fennec-cold:
     run-on-projects: []
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-tp6m-cold-26
             - --app=fennec
             - --binary=org.mozilla.fennec_aurora
 
+raptor-tp6m-27-fennec-cold:
+    description: "Raptor tp6m-27 cold page-load on Fennec"
+    max-run-time: 2700
+    try-name: raptor-tp6m-27-fennec-cold
+    treeherder-symbol: Rap(tp6m-c-27)
+    target: target.apk
+    run-on-projects: []
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-27
+            - --app=fennec
+            - --binary=org.mozilla.fennec_aurora
+
 raptor-tp6m-1-fennec64:
     description: "Raptor tp6m-1 on Fennec64"
     max-run-time: 2700
     try-name: raptor-tp6m-1-fennec64
     treeherder-symbol: Rap(tp6m-1-f64)
     target:
         by-test-platform:
             android-hw-p2-8-0-android-aarch64.*:
@@ -4325,16 +4383,38 @@ raptor-tp6m-26-refbrow-cold:
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-tp6m-cold-26
             - --app=refbrow
             - --binary-path=org.mozilla.reference.browser.raptor
             - --activity=org.mozilla.reference.browser.BrowserTestActivity
 
+raptor-tp6m-27-refbrow-cold:
+    description: "Raptor tp6m-27 cold page-load on the reference browser"
+    try-name: raptor-tp6m-27-refbrow-cold
+    treeherder-symbol: Rap-refbrow(tp6m-c-27)
+    run-on-projects: []
+    e10s: true
+    target:
+        by-test-platform:
+            android-hw.*-aarch64.*/.*:
+                index: project.mobile.reference-browser.v2.raptor.latest.aarch64
+                name: target.apk
+            default:
+                index: project.mobile.reference-browser.v2.raptor.latest.arm
+                name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-27
+            - --app=refbrow
+            - --binary-path=org.mozilla.reference.browser.raptor
+            - --activity=org.mozilla.reference.browser.BrowserTestActivity
+
 raptor-ares6-firefox:
     description: "Raptor Ares6 on Firefox"
     variants: ["fission"]
     try-name: raptor-ares6-firefox
     treeherder-symbol: Rap(ares6)
     max-run-time: 2500
     tier: 3
     mozharness:
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -495,16 +495,17 @@ android-hw-arm7-raptor:
     - raptor-tp6m-19-fenix-cold
     - raptor-tp6m-20-fenix-cold
     - raptor-tp6m-21-fenix-cold
     - raptor-tp6m-22-fenix-cold
     - raptor-tp6m-23-fenix-cold
     - raptor-tp6m-24-fenix-cold
     - raptor-tp6m-25-fenix-cold
     - raptor-tp6m-26-fenix-cold
+    - raptor-tp6m-27-fenix-cold
     - raptor-tp6m-1-geckoview-cold
     - raptor-tp6m-2-geckoview-cold
     - raptor-tp6m-3-geckoview-cold
     - raptor-tp6m-4-geckoview-cold
     - raptor-tp6m-5-geckoview-cold
     - raptor-tp6m-6-geckoview-cold
     - raptor-tp6m-7-geckoview-cold
     - raptor-tp6m-8-geckoview-cold
@@ -521,16 +522,17 @@ android-hw-arm7-raptor:
     - raptor-tp6m-19-geckoview-cold
     - raptor-tp6m-20-geckoview-cold
     - raptor-tp6m-21-geckoview-cold
     - raptor-tp6m-22-geckoview-cold
     - raptor-tp6m-23-geckoview-cold
     - raptor-tp6m-24-geckoview-cold
     - raptor-tp6m-25-geckoview-cold
     - raptor-tp6m-26-geckoview-cold
+    - raptor-tp6m-27-geckoview-cold
     - raptor-tp6m-1-fennec64
     - raptor-tp6m-2-fennec64
     - raptor-tp6m-3-fennec64
     - raptor-tp6m-4-fennec64
     - raptor-tp6m-5-fennec64
     - raptor-tp6m-6-fennec64
     - raptor-tp6m-7-fennec64
     - raptor-tp6m-8-fennec64
@@ -557,16 +559,17 @@ android-hw-arm7-raptor:
     - raptor-tp6m-19-fennec-cold
     - raptor-tp6m-20-fennec-cold
     - raptor-tp6m-21-fennec-cold
     - raptor-tp6m-22-fennec-cold
     - raptor-tp6m-23-fennec-cold
     - raptor-tp6m-24-fennec-cold
     - raptor-tp6m-25-fennec-cold
     - raptor-tp6m-26-fennec-cold
+    - raptor-tp6m-27-fennec-cold
     - raptor-tp6m-1-fennec64-cold
     - raptor-tp6m-2-fennec64-cold
     - raptor-tp6m-3-fennec64-cold
     - raptor-tp6m-4-fennec64-cold
     - raptor-tp6m-5-fennec64-cold
     - raptor-tp6m-6-fennec64-cold
     - raptor-tp6m-7-fennec64-cold
     - raptor-tp6m-8-fennec64-cold
@@ -597,16 +600,17 @@ android-hw-arm7-raptor:
     - raptor-tp6m-19-refbrow-cold
     - raptor-tp6m-20-refbrow-cold
     - raptor-tp6m-21-refbrow-cold
     - raptor-tp6m-22-refbrow-cold
     - raptor-tp6m-23-refbrow-cold
     - raptor-tp6m-24-refbrow-cold
     - raptor-tp6m-25-refbrow-cold
     - raptor-tp6m-26-refbrow-cold
+    - raptor-tp6m-27-refbrow-cold
 
 android-hw-aarch64-raptor:
     - raptor-speedometer-geckoview
     - raptor-speedometer-refbrow
     - raptor-youtube-playback-fenix
     - raptor-youtube-playback-geckoview
     - raptor-tp6m-1-geckoview
     - raptor-tp6m-2-geckoview
@@ -659,16 +663,17 @@ android-hw-aarch64-raptor:
     - raptor-tp6m-19-fenix-cold
     - raptor-tp6m-20-fenix-cold
     - raptor-tp6m-21-fenix-cold
     - raptor-tp6m-22-fenix-cold
     - raptor-tp6m-23-fenix-cold
     - raptor-tp6m-24-fenix-cold
     - raptor-tp6m-25-fenix-cold
     - raptor-tp6m-26-fenix-cold
+    - raptor-tp6m-27-fenix-cold
     - raptor-tp6m-1-geckoview-cold
     - raptor-tp6m-2-geckoview-cold
     - raptor-tp6m-3-geckoview-cold
     - raptor-tp6m-4-geckoview-cold
     - raptor-tp6m-5-geckoview-cold
     - raptor-tp6m-6-geckoview-cold
     - raptor-tp6m-7-geckoview-cold
     - raptor-tp6m-8-geckoview-cold
@@ -685,16 +690,17 @@ android-hw-aarch64-raptor:
     - raptor-tp6m-19-geckoview-cold
     - raptor-tp6m-20-geckoview-cold
     - raptor-tp6m-21-geckoview-cold
     - raptor-tp6m-22-geckoview-cold
     - raptor-tp6m-23-geckoview-cold
     - raptor-tp6m-24-geckoview-cold
     - raptor-tp6m-25-geckoview-cold
     - raptor-tp6m-26-geckoview-cold
+    - raptor-tp6m-27-geckoview-cold
     - raptor-tp6m-1-fennec-cold
     - raptor-tp6m-2-fennec-cold
     - raptor-tp6m-3-fennec-cold
     - raptor-tp6m-4-fennec-cold
     - raptor-tp6m-5-fennec-cold
     - raptor-tp6m-6-fennec-cold
     - raptor-tp6m-7-fennec-cold
     - raptor-tp6m-8-fennec-cold
@@ -711,16 +717,17 @@ android-hw-aarch64-raptor:
     - raptor-tp6m-19-fennec-cold
     - raptor-tp6m-20-fennec-cold
     - raptor-tp6m-21-fennec-cold
     - raptor-tp6m-22-fennec-cold
     - raptor-tp6m-23-fennec-cold
     - raptor-tp6m-24-fennec-cold
     - raptor-tp6m-25-fennec-cold
     - raptor-tp6m-26-fennec-cold
+    - raptor-tp6m-27-fennec-cold
     - raptor-tp6m-1-fennec64-cold
     - raptor-tp6m-2-fennec64-cold
     - raptor-tp6m-3-fennec64-cold
     - raptor-tp6m-4-fennec64-cold
     - raptor-tp6m-5-fennec64-cold
     - raptor-tp6m-6-fennec64-cold
     - raptor-tp6m-7-fennec64-cold
     - raptor-tp6m-8-fennec64-cold
@@ -751,16 +758,17 @@ android-hw-aarch64-raptor:
     - raptor-tp6m-19-refbrow-cold
     - raptor-tp6m-20-refbrow-cold
     - raptor-tp6m-21-refbrow-cold
     - raptor-tp6m-22-refbrow-cold
     - raptor-tp6m-23-refbrow-cold
     - raptor-tp6m-24-refbrow-cold
     - raptor-tp6m-25-refbrow-cold
     - raptor-tp6m-26-refbrow-cold
+    - raptor-tp6m-27-refbrow-cold
 
 android-hw-arm7-raptor-power:
     - raptor-speedometer-geckoview-power
     - raptor-scn-power-idle-fenix
     - raptor-scn-power-idle-fennec
     - raptor-scn-power-idle-geckoview
     - raptor-scn-power-idle-refbrow
     - raptor-scn-power-idle-bg-fenix
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@@ -63,16 +63,17 @@
 [include:tests/tp6/mobile/raptor-tp6m-cold-19.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-20.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-21.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-22.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-23.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-24.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-25.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-26.ini]
+[include:tests/tp6/mobile/raptor-tp6m-cold-27.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-1-fennec64.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-2-fennec64.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-3-fennec64.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-4-fennec64.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-5-fennec64.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-6-fennec64.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-7-fennec64.ini]
 [include:tests/tp6/mobile/raptor-tp6m-cold-8-fennec64.ini]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/tp6/mobile/raptor-tp6m-cold-27.ini
@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-27
+
+[DEFAULT]
+type =  pageload
+test_url = https://edition.cnn.com
+playback = mitmproxy-android
+playback_version = 4.0.4
+playback_binary_manifest = mitmproxy-rel-bin-4.0.4-{platform}.manifest
+playback_pageset_manifest = mitm4-motog5-gve-cnn.manifest
+playback_recordings = android-cnn.mp
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-cnn-fenix-cold]
+apps = fenix
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-cnn-geckoview-cold]
+apps = geckoview
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-cnn-fennec-cold]
+apps = fennec
+measure = fnbpaint, dcf, loadtime
+alert_on = loadtime
+
+[raptor-tp6m-cnn-refbrow-cold]
+apps = refbrow
+measure = fnbpaint, fcp, dcf, loadtime