Bug 1529210 Add new Raptor tests in tp6m-5 r=davehunt
authoralexandru.ionescu <alexandru.ionescu@softvision.ro>
Fri, 01 Mar 2019 14:46:36 +0000
changeset 519833 7f3e90c4b9c2b308afaaf0b40c3ea2d2e8fcc947
parent 519832 214aa79533cd44a15f8a45ab9a3eee05dd957047
child 519834 93f7dc3084a1350e5c2c21d599ec6634ebe0ec8f
push id10862
push userffxbld-merge
push dateMon, 11 Mar 2019 13:01:11 +0000
treeherdermozilla-beta@a2e7f5c935da [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdavehunt
bugs1529210
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1529210 Add new Raptor tests in tp6m-5 r=davehunt Differential Revision: https://phabricator.services.mozilla.com/D20815
taskcluster/ci/test/raptor.yml
taskcluster/ci/test/test-sets.yml
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-allrecipes.manifest
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-amazon-search.manifest
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-espn.manifest
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-web-de.manifest
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-wikipedia.manifest
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-youtube-watch.manifest
testing/raptor/raptor/raptor.ini
testing/raptor/raptor/tests/raptor-tp6m-5.ini
testing/raptor/raptor/tests/raptor-tp6m-8.ini
testing/raptor/webext/raptor/manifest.json
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -401,16 +401,51 @@ raptor-tp6m-2-geckoview:
             default: ['try', 'trunk', 'mozilla-beta']
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-tp6m-2
             - --app=geckoview
             - --binary=org.mozilla.geckoview_example
 
+raptor-tp6m-5-geckoview:
+    description: "Raptor tp6m-5 on Geckoview"
+    try-name: raptor-tp6m-5-geckoview
+    treeherder-symbol: Rap(tp6m-5)
+    run-on-projects:
+        by-test-platform:
+            android-hw-p2-8-0-arm7.*: ['try', 'mozilla-central']
+            android-hw-g5.*: ['try', 'mozilla-central']
+            default: ['try', 'trunk', 'mozilla-beta']
+    target: geckoview_example.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-5
+            - --app=geckoview
+            - --binary=org.mozilla.geckoview_example
+            - --activity=GeckoViewActivity
+
+raptor-tp6m-8-geckoview:
+    description: "Raptor tp6m-8 on Geckoview"
+    try-name: raptor-tp6m-8-geckoview
+    treeherder-symbol: Rap(tp6m-8)
+    run-on-projects:
+        by-test-platform:
+            android-hw-p2-8-0-arm7.*: ['try', 'mozilla-central']
+            android-hw-g5.*: ['try', 'mozilla-central']
+            default: ['try', 'trunk', 'mozilla-beta']
+    target: geckoview_example.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-8
+            - --app=geckoview
+            - --binary=org.mozilla.geckoview_example
+
 raptor-speedometer-firefox:
     description: "Raptor Speedometer on Firefox"
     try-name: raptor-speedometer-firefox
     treeherder-symbol: Rap(sp)
     mozharness:
         extra-options:
             - --test=raptor-speedometer
 
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -408,21 +408,25 @@ android-hw-arm7-debug-unittests:
 
 android-hw-aarch64-opt-unittests:
     - jittest
 
 android-hw-arm7-raptor:
     - raptor-speedometer-geckoview
     - raptor-tp6m-1-geckoview
     - raptor-tp6m-2-geckoview
+    - raptor-tp6m-5-geckoview
+    - raptor-tp6m-8-geckoview
 
 android-hw-aarch64-raptor:
     - raptor-speedometer-geckoview
     - raptor-tp6m-1-geckoview
     - raptor-tp6m-2-geckoview
+    - raptor-tp6m-5-geckoview
+    - raptor-tp6m-8-geckoview
 
 android-hw-arm7-raptor-power:
     - raptor-speedometer-geckoview-power
 
 android-hw-aarch64-raptor-power:
     - raptor-speedometer-geckoview-power
 
 android-hw-arm7-raptor-nightly:
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-allrecipes.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 9226722,
+    "visibility": "public",
+    "digest": "747dd05276ca4cfbba414548b734f068f6033fbcfb49745f2ac45c5538ae81029a7818152f67162a465be89f4d0fb7486dfce76dae37667a6abb10916fa6b85d",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-tp6m-allrecipes.zip",
+    "unpack": true
+  }
+]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-amazon-search.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 1683753,
+    "visibility": "public",
+    "digest": "acc3fb1944d82a8d544f3395398e334ff5f3e152852bf7db8a1125911b70a404ca04483baf25ccac6372d4d57f384e23ea665d38fa3991dfae72f0f9f5483b0e",
+    "algorithm": "sha512",
+    "filename": "android-amazon-search.zip",
+    "unpack": true
+  }
+]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-espn.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 3549626,
+    "visibility": "public",
+    "digest": "35c6bd4f2de96d3df7134a50ab17beba21e2219a82ecbe89555271f21f75fa97ab6adf208d358689fad155a282ed3ed9f6d5948b98df4246180054404df306e8",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-tp6m-espn.zip",
+    "unpack": true
+  }
+]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-web-de.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 5246741,
+    "visibility": "public",
+    "digest": "a0224cf7279b6144be401c49dc099e8c9ff51b2d620f2ad2f31761c02e1a6546791f84bd964cb441076242f33cb4088d9324c2d8b097bffcc0352edef25a568a",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-tp6m-web-de.zip",
+    "unpack": true
+  }
+]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-wikipedia.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 4487675,
+    "visibility": "public",
+    "digest": "66b6d0dfdb06434b596c1e5cd631c2927501d54cc1d5493742f87672734987a96631ea3ebc2e2bc4600de345ea3a7d20be0f85077b9b6919484c698e2612da69",
+    "algorithm": "sha512",
+    "filename": "android-wikipedia.zip",
+    "unpack": true
+  }
+]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-youtube-watch.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 1920383,
+    "visibility": "public",
+    "digest": "3de23f3a704db9b3b775776a3c32cd9ad3fa56f26d8a9c8951056a2a19f491c9f36dc810ae452805ee5561591aba8467ae35f6b1be330613a0db7aa7d92d4acd",
+    "algorithm": "sha512",
+    "filename": "android-youtube-watch.zip",
+    "unpack": true
+  }
+]
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@@ -11,16 +11,18 @@
 [include:tests/raptor-tp6-10.ini]
 
 # raptor pageload binast tests desktop
 [include:tests/raptor-tp6-binast-1.ini]
 
 # raptor pageload tests mobile
 [include:tests/raptor-tp6m-1.ini]
 [include:tests/raptor-tp6m-2.ini]
+[include:tests/raptor-tp6m-5.ini]
+[include:tests/raptor-tp6m-8.ini]
 
 # raptor benchmark tests
 [include:tests/raptor-assorted-dom.ini]
 [include:tests/raptor-motionmark-animometer.ini]
 [include:tests/raptor-motionmark-htmlsuite.ini]
 [include:tests/raptor-speedometer.ini]
 [include:tests/raptor-stylebench.ini]
 [include:tests/raptor-sunspider.ini]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-5.ini
@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-5
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+page_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+
+[raptor-tp6m-amazon-search-geckoview]
+apps = geckoview
+test_url = https://www.amazon.com/s/ref=nb_sb_noss_2/139-6317191-5622045?url=search-alias%3Daps&field-keywords=mobile+phone
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-amazon-search.manifest
+playback_recordings = android-amazon-search.mp
+measure = fnbpaint, fcp, dcf, ttfi, loadtime
+
+[raptor-tp6m-wikipedia-geckoview]
+apps = geckoview
+test_url = https://en.m.wikipedia.org/wiki/Main_Page
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-wikipedia.manifest
+playback_recordings = android-wikipedia.mp
+measure = fnbpaint, fcp, dcf, ttfi, loadtime
+
+[raptor-tp6m-youtube-geckoview]
+apps = geckoview
+test_url = https://www.youtube.com/watch?v=COU5T-Wafa4
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-youtube-watch.manifest
+playback_recordings = android-youtube-watch.mp
+measure = fnbpaint, fcp, dcf, ttfi, loadtime
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-8.ini
@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-8
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+page_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+
+[raptor-tp6m-allrecipes-geckoview]
+apps = geckoview
+test_url = https://www.allrecipes.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-allrecipes.manifest
+playback_recordings = android-allrecipes.mp
+measure = fnbpaint, fcp, dcf, ttfi, loadtime
+
+[raptor-tp6m-espn-geckoview]
+apps = geckoview
+test_url = http://www.espn.com/nba/story/_/page/allstarweekend25788027/the-comparison-lebron-james-michael-jordan-their-own-words
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-espn.manifest
+playback_recordings = android-espn.mp
+measure = fnbpaint, fcp, dcf, ttfi, loadtime
+
+[raptor-tp6m-web-de-geckoview]
+apps = geckoview
+test_url = https://web.de/magazine/politik/politologe-glaubt-grossen-koalition-herbst-knallen-33563566
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-web-de.manifest
+playback_recordings = android-web-de.mp
+measure = fnbpaint, fcp, dcf, ttfi, loadtime
--- a/testing/raptor/webext/raptor/manifest.json
+++ b/testing/raptor/webext/raptor/manifest.json
@@ -9,33 +9,36 @@
   "version": "0.1",
   "description": "Performance measurement framework prototype",
   "background": {
     "scripts": ["auto_gen_test_config.js", "runner.js"]
   },
   "content_scripts": [
     {
       "matches": [
+                  "*://*.allrecipes.com/*",
                   "*://*.apple.com/*",
                   "*://*.amazon.com/*",
                   "*://*.bing.com/*",
                   "*://*.ebay.com/*",
                   "*://*.ebay-kleinanzeigen.de/*",
+                  "*://*.espn.com/*",
                   "*://*.facebook.com/*",
                   "*://*.google.com/*",
                   "*://*.imdb.com/*",
                   "*://*.imgur.com/*",
                   "*://*.instagram.com/*",
                   "*://*.microsoft.com/*",
                   "*://*.paypal.com/*",
                   "*://*.pinterest.com/*",
                   "*://*.reddit.com/*",
                   "*://*.tumblr.com/*",
                   "*://*.twitter.com/*",
                   "*://*.vice.com/*",
+                  "*://*.web.de/*",
                   "*://*.wikia.com/*",
                   "*://*.wikipedia.org/*",
                   "*://*.yahoo.com/*",
                   "*://*.youtube.com/*",
                   "*://*.yandex.ru/*"
                   ],
       "js": ["measure.js"]
     },