Bug 1545175 - Raptor test definitions and taskcluster configs for cold page-load tests on Fenix; r=Bebe,jmaher
authorRob Wood <rwood@mozilla.com>
Thu, 18 Apr 2019 19:34:10 +0000
changeset 470141 050964a5e03e9888993e610c10d96ffc68cd9883
parent 470140 5a890f8d88d5f45309ba1fc54e0ad610f72b6948
child 470142 be77d454cbe7e8cf3d619b56d87d399699bd7edd
push id112843
push useraiakab@mozilla.com
push dateFri, 19 Apr 2019 09:50:22 +0000
treeherdermozilla-inbound@c06f27cbfe40 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersBebe, jmaher
bugs1545175
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1545175 - Raptor test definitions and taskcluster configs for cold page-load tests on Fenix; r=Bebe,jmaher Differential Revision: https://phabricator.services.mozilla.com/D27964
taskcluster/ci/test/raptor.yml
taskcluster/ci/test/test-sets.yml
testing/raptor/raptor/raptor.ini
testing/raptor/raptor/tests/raptor-tp6m-cold-1.ini
testing/raptor/raptor/tests/raptor-tp6m-cold-10.ini
testing/raptor/raptor/tests/raptor-tp6m-cold-11.ini
testing/raptor/raptor/tests/raptor-tp6m-cold-12.ini
testing/raptor/raptor/tests/raptor-tp6m-cold-13.ini
testing/raptor/raptor/tests/raptor-tp6m-cold-14.ini
testing/raptor/raptor/tests/raptor-tp6m-cold-2.ini
testing/raptor/raptor/tests/raptor-tp6m-cold-3.ini
testing/raptor/raptor/tests/raptor-tp6m-cold-4.ini
testing/raptor/raptor/tests/raptor-tp6m-cold-5.ini
testing/raptor/raptor/tests/raptor-tp6m-cold-6.ini
testing/raptor/raptor/tests/raptor-tp6m-cold-7.ini
testing/raptor/raptor/tests/raptor-tp6m-cold-8.ini
testing/raptor/raptor/tests/raptor-tp6m-cold-9.ini
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -747,16 +747,254 @@ raptor-tp6m-10-fennec:
     run-on-projects: ['try']
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-tp6m-10-fennec
             - --app=fennec
             - --binary=org.mozilla.fennec_aurora
 
+raptor-tp6m-1-fenix-cold:
+    description: "Raptor tp6m-1 cold page-load on Fenix"
+    try-name: raptor-tp6m-1-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-1)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-1
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-2-fenix-cold:
+    description: "Raptor tp6m-2 cold page-load on Fenix"
+    try-name: raptor-tp6m-2-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-2)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-2
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-3-fenix-cold:
+    description: "Raptor tp6m-3 cold page-load on Fenix"
+    try-name: raptor-tp6m-3-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-3)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-3
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-4-fenix-cold:
+    description: "Raptor tp6m-4 cold page-load on Fenix"
+    try-name: raptor-tp6m-4-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-4)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-4
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-5-fenix-cold:
+    description: "Raptor tp6m-5 cold page-load on Fenix"
+    try-name: raptor-tp6m-5-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-5)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-5
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-6-fenix-cold:
+    description: "Raptor tp6m-6 cold page-load on Fenix"
+    try-name: raptor-tp6m-6-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-6)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-6
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-7-fenix-cold:
+    description: "Raptor tp6m-7 cold page-load on Fenix"
+    try-name: raptor-tp6m-7-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-7)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-7
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-8-fenix-cold:
+    description: "Raptor tp6m-8 cold page-load on Fenix"
+    try-name: raptor-tp6m-8-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-8)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-8
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-9-fenix-cold:
+    description: "Raptor tp6m-9 cold page-load on Fenix"
+    try-name: raptor-tp6m-9-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-9)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-9
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-10-fenix-cold:
+    description: "Raptor tp6m-10 cold page-load on Fenix"
+    try-name: raptor-tp6m-10-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-10)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-10
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-11-fenix-cold:
+    description: "Raptor tp6m-11 cold page-load on Fenix"
+    try-name: raptor-tp6m-11-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-11)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-11
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-12-fenix-cold:
+    description: "Raptor tp6m-12 cold page-load on Fenix"
+    try-name: raptor-tp6m-12-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-12)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-12
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-13-fenix-cold:
+    description: "Raptor tp6m-13 cold page-load on Fenix"
+    try-name: raptor-tp6m-13-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-13)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-13
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
+raptor-tp6m-14-fenix-cold:
+    description: "Raptor tp6m-14 cold page-load on Fenix"
+    try-name: raptor-tp6m-14-fenix-cold
+    treeherder-symbol: Rap-fenix(tp6m-c-14)
+    run-on-projects: ['try']
+    e10s: true
+    target:
+        index: project.mobile.fenix.branch.master.latest.greenfield.arm-release-raptor
+        name: target.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-cold-14
+            - --app=fenix
+            - --binary-path=org.mozilla.fenix
+            - --activity=browser.BrowserPerformanceTestActivity
+
 raptor-speedometer-firefox:
     description: "Raptor Speedometer on Firefox"
     try-name: raptor-speedometer-firefox
     treeherder-symbol: Rap(sp)
     mozharness:
         extra-options:
             - --test=raptor-speedometer
 
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -465,16 +465,30 @@ android-hw-arm7-raptor:
     - raptor-tp6m-3-refbrow
     - raptor-tp6m-4-refbrow
     - raptor-tp6m-5-refbrow
     - raptor-tp6m-6-refbrow
     - raptor-tp6m-7-refbrow
     - raptor-tp6m-8-refbrow
     - raptor-tp6m-9-refbrow
     - raptor-tp6m-10-refbrow
+    - raptor-tp6m-1-fenix-cold
+    - raptor-tp6m-2-fenix-cold
+    - raptor-tp6m-3-fenix-cold
+    - raptor-tp6m-4-fenix-cold
+    - raptor-tp6m-5-fenix-cold
+    - raptor-tp6m-6-fenix-cold
+    - raptor-tp6m-7-fenix-cold
+    - raptor-tp6m-8-fenix-cold
+    - raptor-tp6m-9-fenix-cold
+    - raptor-tp6m-10-fenix-cold
+    - raptor-tp6m-11-fenix-cold
+    - raptor-tp6m-12-fenix-cold
+    - raptor-tp6m-13-fenix-cold
+    - raptor-tp6m-14-fenix-cold
 
 android-hw-aarch64-raptor:
     - raptor-speedometer-geckoview
     - raptor-speedometer-refbrow
     - raptor-tp6m-1-geckoview
     - raptor-tp6m-2-geckoview
     - raptor-tp6m-3-geckoview
     - raptor-tp6m-4-geckoview
@@ -490,16 +504,30 @@ android-hw-aarch64-raptor:
     - raptor-tp6m-3-refbrow
     - raptor-tp6m-4-refbrow
     - raptor-tp6m-5-refbrow
     - raptor-tp6m-6-refbrow
     - raptor-tp6m-7-refbrow
     - raptor-tp6m-8-refbrow
     - raptor-tp6m-9-refbrow
     - raptor-tp6m-10-refbrow
+    - raptor-tp6m-1-fenix-cold
+    - raptor-tp6m-2-fenix-cold
+    - raptor-tp6m-3-fenix-cold
+    - raptor-tp6m-4-fenix-cold
+    - raptor-tp6m-5-fenix-cold
+    - raptor-tp6m-6-fenix-cold
+    - raptor-tp6m-7-fenix-cold
+    - raptor-tp6m-8-fenix-cold
+    - raptor-tp6m-9-fenix-cold
+    - raptor-tp6m-10-fenix-cold
+    - raptor-tp6m-11-fenix-cold
+    - raptor-tp6m-12-fenix-cold
+    - raptor-tp6m-13-fenix-cold
+    - raptor-tp6m-14-fenix-cold
 
 android-hw-arm7-raptor-power:
     - raptor-speedometer-geckoview-power
 
 android-hw-aarch64-raptor-power:
     - raptor-speedometer-geckoview-power
 
 android-hw-arm7-raptor-nightly:
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@@ -23,16 +23,29 @@
 [include:tests/raptor-tp6m-6.ini]
 [include:tests/raptor-tp6m-7.ini]
 [include:tests/raptor-tp6m-8.ini]
 [include:tests/raptor-tp6m-9.ini]
 [include:tests/raptor-tp6m-10.ini]
 
 # raptor cold pageload tests mobile
 [include:tests/raptor-tp6m-cold-1.ini]
+[include:tests/raptor-tp6m-cold-2.ini]
+[include:tests/raptor-tp6m-cold-3.ini]
+[include:tests/raptor-tp6m-cold-4.ini]
+[include:tests/raptor-tp6m-cold-5.ini]
+[include:tests/raptor-tp6m-cold-6.ini]
+[include:tests/raptor-tp6m-cold-7.ini]
+[include:tests/raptor-tp6m-cold-8.ini]
+[include:tests/raptor-tp6m-cold-9.ini]
+[include:tests/raptor-tp6m-cold-10.ini]
+[include:tests/raptor-tp6m-cold-11.ini]
+[include:tests/raptor-tp6m-cold-12.ini]
+[include:tests/raptor-tp6m-cold-13.ini]
+[include:tests/raptor-tp6m-cold-14.ini]
 
 # raptor benchmark tests
 [include:tests/raptor-assorted-dom.ini]
 [include:tests/raptor-motionmark-animometer.ini]
 [include:tests/raptor-motionmark-htmlsuite.ini]
 [include:tests/raptor-speedometer.ini]
 [include:tests/raptor-stylebench.ini]
 [include:tests/raptor-sunspider.ini]
--- a/testing/raptor/raptor/tests/raptor-tp6m-cold-1.ini
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-1.ini
@@ -3,17 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # raptor tp6m-cold-1
 
 [DEFAULT]
 type =  pageload
 playback = mitmproxy-android
 playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
-browser_cycles = 10
+browser_cycles = 15
 unit = ms
 lower_is_better = true
 alert_threshold = 2.0
 page_timeout = 60000
 alert_on = fcp, loadtime
 cold = true
 
 [raptor-tp6m-cold-amazon-geckoview]
@@ -25,14 +25,21 @@ measure = fnbpaint, fcp, dcf, loadtime
 
 [raptor-tp6m-cold-facebook-geckoview]
 apps = geckoview
 test_url = https://m.facebook.com
 playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-facebook.manifest
 playback_recordings = android-facebook.mp
 measure = fnbpaint, fcp, dcf, loadtime
 
-[raptor-tp6m-cold-google-geckoview]
-apps = geckoview
-test_url = https://www.google.com
-playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-google.manifest
-playback_recordings = android-google.mp
+[raptor-tp6m-cold-amazon-fenix]
+apps = fenix
+test_url = https://www.amazon.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-amazon.manifest
+playback_recordings = android-amazon.mp
 measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-cold-facebook-fenix]
+apps = fenix
+test_url = https://m.facebook.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-facebook.manifest
+playback_recordings = android-facebook.mp
+measure = fnbpaint, fcp, dcf, loadtime
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-10.ini
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-10
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-bbc-fenix]
+apps = fenix
+test_url = https://www.bbc.com/news/business-47245877
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-bbc.manifest
+playback_recordings = mitmproxy-recordings-raptor-tp6m-bbc.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-stackoverflow-fenix]
+apps = fenix
+test_url = https://stackoverflow.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-stackoverflow.manifest
+playback_recordings = mitmproxy-recordings-raptor-tp6m-stackoverflow.mp
+measure = fnbpaint, fcp, dcf, loadtime
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-11.ini
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-11
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-microsoft-support-fenix]
+apps = fenix
+test_url = https://support.microsoft.com/en-us
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-microsoft-support.manifest
+playback_recordings = android-microsoft-support.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-jianshu-fenix]
+apps = fenix
+test_url = https://www.jianshu.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-jianshu.manifest
+playback_recordings = android-jianshu.mp
+measure = fnbpaint, fcp, dcf, loadtime
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-12.ini
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-12
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-imdb-fenix]
+apps = fenix
+test_url = https://m.imdb.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-imdb.manifest
+playback_recordings = android-imdb.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-allrecipes-fenix]
+apps = fenix
+test_url = https://www.allrecipes.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-allrecipes.manifest
+playback_recordings = android-allrecipes.mp
+measure = fnbpaint, fcp, dcf, loadtime
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-13.ini
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-13
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-espn-fenix]
+apps = fenix
+test_url = http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-espn.manifest
+playback_recordings = android-espn.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-web-de-fenix]
+apps = fenix
+test_url = https://web.de/magazine/politik/politologe-glaubt-grossen-koalition-herbst-knallen-33563566
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-web-de.manifest
+playback_recordings = android-web-de.mp
+measure = fnbpaint, fcp, dcf, loadtime
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-14.ini
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-14
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-facebook-cristiano-fenix]
+apps = fenix
+test_url = https://m.facebook.com/Cristiano
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-facebook-cristiano.manifest
+playback_recordings = android-facebook-cristiano.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-aframeio-animation-fenix]
+apps = fenix
+test_url = https://aframe.io/examples/showcase/animation
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-aframeio-animation.manifest
+playback_recordings = android-aframeio-animation.mp
+measure = fnbpaint, fcp, dcf, loadtime
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-2.ini
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-2
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-google-fenix]
+apps = fenix
+test_url = https://www.google.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-google.manifest
+playback_recordings = android-google.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-youtube-fenix]
+apps = fenix
+test_url = https://www.youtube.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-youtube.manifest
+playback_recordings = android-youtube.mp
+measure = fnbpaint, fcp, dcf, loadtime
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-3.ini
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-3
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-instagram-fenix]
+apps = fenix
+test_url = https://www.instagram.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-instagram-mobile.manifest
+playback_recordings = instagram-mobile.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-bing-fenix]
+apps = fenix
+test_url = https://www.bing.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-bing-mobile.manifest
+playback_recordings = bing-mobile.mp
+measure = fnbpaint, fcp, dcf, loadtime
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-4.ini
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-4
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-bing-restaurants-fenix]
+apps = fenix
+test_url = https://www.bing.com/search?q=restaurants
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-bing-restaurants-mobile.manifest
+playback_recordings = bing-restaurants-mobile.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-ebay-kleinanzeigen-fenix]
+apps = fenix
+test_url = https://m.ebay-kleinanzeigen.de
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-ebay-kleinanzeigen-mobile.manifest
+playback_recordings = ebay-kleinanzeigen-mobile.mp
+measure = fnbpaint, fcp, dcf, loadtime
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-5.ini
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-5
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-ebay-kleinanzeigen-search-fenix]
+apps = fenix
+test_url = https://m.ebay-kleinanzeigen.de/s-anzeigen/auf-zeit-wg-berlin/zimmer/c199-l3331
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-ebay-kleinanzeigen-search-mobile.manifest
+playback_recordings = ebay-kleinanzeigen-search-mobile.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-google-maps-fenix]
+apps = fenix
+test_url = https://www.google.com/maps?force=pwa
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-google-maps.manifest
+playback_recordings = google_maps_mobile.mp
+measure = fnbpaint, fcp, dcf, loadtime
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-6.ini
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-6
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-google-restaurants-fenix]
+apps = fenix
+test_url = https://www.google.com/search?q=restaurants+near+me
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-google-restaurants.manifest
+playback_recordings = google-search-restaurants-mobile.mp
+measure = fnbpaint, fcp, dcf, loadtime
+disabled = Bug 1533283 Intermittent timeouts running raptor-tp6m-google-restaurants-geckoview
+
+[raptor-tp6m-amazon-search-fenix]
+apps = fenix
+test_url = https://www.amazon.com/s/ref=nb_sb_noss_2/139-6317191-5622045?url=search-alias%3Daps&field-keywords=mobile+phone
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-amazon-search.manifest
+playback_recordings = android-amazon-search.mp
+measure = fnbpaint, fcp, dcf, loadtime
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-7.ini
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-7
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-wikipedia-fenix]
+apps = fenix
+test_url = https://en.m.wikipedia.org/wiki/Main_Page
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-wikipedia.manifest
+playback_recordings = android-wikipedia.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-youtube-watch-fenix]
+apps = fenix
+test_url = https://www.youtube.com/watch?v=COU5T-Wafa4
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-youtube-watch.manifest
+playback_recordings = android-youtube-watch.mp
+measure = fnbpaint, fcp, dcf, loadtime
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-8.ini
@@ -0,0 +1,32 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-8
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-booking-fenix]
+apps = fenix
+test_url = https://www.booking.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-booking.manifest
+playback_recordings = android-booking.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-cnn-fenix]
+apps = fenix
+test_url = https://edition.cnn.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-cnn.manifest
+playback_recordings = android-cnn.mp
+measure = fnbpaint, fcp, dcf, loadtime
+disabled = Bug 1533287 Intermittent timeouts running raptor-tp6m-cnn-geckoview
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-cold-9.ini
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-cold-9
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+browser_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6m-cnn-ampstories-fenix]
+apps = fenix
+test_url = https://edition.cnn.com/ampstories/us/why-hurricane-michael-is-a-monster-unlike-any-other
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-cnn-ampstories.manifest
+playback_recordings = android-cnn-ampstories.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6m-reddit-fenix]
+apps = fenix
+test_url = https://www.reddit.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-reddit.manifest
+playback_recordings = mitmproxy-recordings-raptor-tp6m-reddit.mp
+measure = fnbpaint, fcp, dcf, loadtime