Bug 1503990 add additional pages to tp6 r=rwood
authorflorin.strugariu <bebe@mozilla.ro>
Tue, 20 Nov 2018 01:21:38 +0000
changeset 503512 36aafe5071875107b0c68d4c455bd294db9dac0b
parent 503511 dba5cff2fd640bb82eb15988b095679262f5b621
child 503513 5884c1ce66969e88d18cb453a2e92304d1ddb339
push id10290
push userffxbld-merge
push dateMon, 03 Dec 2018 16:23:23 +0000
treeherdermozilla-beta@700bed2445e6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrwood
bugs1503990
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1503990 add additional pages to tp6 r=rwood Differential Revision: https://phabricator.services.mozilla.com/D11058
taskcluster/ci/test/raptor.yml
taskcluster/ci/test/test-sets.yml
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6-3.manifest
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6-4.manifest
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6-5.manifest
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6-6.manifest
testing/raptor/raptor/raptor.ini
testing/raptor/raptor/tests/raptor-tp6-3.ini
testing/raptor/raptor/tests/raptor-tp6-4.ini
testing/raptor/raptor/tests/raptor-tp6-5.ini
testing/raptor/raptor/tests/raptor-tp6-6.ini
testing/raptor/webext/raptor/manifest.json
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -80,16 +80,120 @@ raptor-tp6-2-chrome:
             linux64.*: 3
             default: 2
     max-run-time: 1800
     mozharness:
         extra-options:
             - --test=raptor-tp6-2
             - --app=chrome
 
+raptor-tp6-3-firefox:
+    description: "Raptor tp6-3 on Firefox"
+    try-name: raptor-tp6-3-firefox
+    treeherder-symbol: Rap(tp6-3)
+    run-on-projects: ['try', 'mozilla-central']
+    tier: 2
+    max-run-time: 1800
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-3
+
+raptor-tp6-3-chrome:
+    description: "Raptor tp6-3 on Chrome"
+    try-name: raptor-tp6-3-chrome
+    treeherder-symbol: Rap-C(tp6-3)
+    run-on-projects: ['try', 'mozilla-central']
+    tier:
+        by-test-platform:
+            linux64.*: 3
+            default: 2
+    max-run-time: 2400
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-3
+            - --app=chrome
+
+raptor-tp6-4-firefox:
+    description: "Raptor tp6-4 on Firefox"
+    try-name: raptor-tp6-4-firefox
+    treeherder-symbol: Rap(tp6-4)
+    run-on-projects: ['try', 'mozilla-central']
+    tier: 2
+    max-run-time: 1200
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-4
+
+raptor-tp6-4-chrome:
+    description: "Raptor tp6-4 on Chrome"
+    try-name: raptor-tp6-4-chrome
+    treeherder-symbol: Rap-C(tp6-4)
+    run-on-projects: ['try', 'mozilla-central']
+    tier:
+        by-test-platform:
+            linux64.*: 3
+            default: 2
+    max-run-time: 1800
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-4
+            - --app=chrome
+
+raptor-tp6-5-firefox:
+    description: "Raptor tp6-5 on Firefox"
+    try-name: raptor-tp6-5-firefox
+    treeherder-symbol: Rap(tp6-5)
+    run-on-projects: ['try', 'mozilla-central']
+    tier: 2
+    max-run-time: 1200
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-5
+
+raptor-tp6-5-chrome:
+    description: "Raptor tp6-5 on Chrome"
+    try-name: raptor-tp6-5-chrome
+    treeherder-symbol: Rap-C(tp6-5)
+    run-on-projects: ['try', 'mozilla-central']
+    tier:
+        by-test-platform:
+            linux64.*: 3
+            default: 2
+    max-run-time: 1800
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-5
+            - --app=chrome
+
+raptor-tp6-6-firefox:
+    description: "Raptor tp6-6 on Firefox"
+    try-name: raptor-tp6-6-firefox
+    treeherder-symbol: Rap(tp6-6)
+    run-on-projects: ['try', 'mozilla-central']
+    tier: 2
+    max-run-time: 1200
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-6
+
+raptor-tp6-6-chrome:
+    description: "Raptor tp6-6 on Chrome"
+    try-name: raptor-tp6-6-chrome
+    treeherder-symbol: Rap-C(tp6-6)
+    run-on-projects: ['try', 'mozilla-central']
+    tier:
+        by-test-platform:
+            linux64.*: 3
+            default: 2
+    max-run-time: 1800
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-6
+            - --app=chrome
+
 raptor-speedometer-firefox:
     description: "Raptor Speedometer on Firefox"
     try-name: raptor-speedometer-firefox
     treeherder-symbol: Rap(sp)
     run-on-projects:
         by-test-platform:
             windows10-64-ux: ['try', 'mozilla-central']
             default: built-projects
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -74,27 +74,35 @@ talos:
     - talos-tp6-stylo-threads
     - talos-tps
     # - talos-h1 Bug 1487031 - Disabled for not finding actionable regressions
     # - talos-h2 Bug 1487031 - Disabled for not finding actionable regressions
 
 raptor-firefox:
     - raptor-tp6-1-firefox
     - raptor-tp6-2-firefox
+    - raptor-tp6-3-firefox
+    - raptor-tp6-4-firefox
+    - raptor-tp6-5-firefox
+    - raptor-tp6-6-firefox
     - raptor-speedometer-firefox
     - raptor-stylebench-firefox
     - raptor-motionmark-htmlsuite-firefox
     - raptor-motionmark-animometer-firefox
     - raptor-webaudio-firefox
     - raptor-sunspider-firefox
     - raptor-wasm-godot-firefox
 
 raptor-chrome:
     - raptor-tp6-1-chrome
     - raptor-tp6-2-chrome
+    - raptor-tp6-3-chrome
+    - raptor-tp6-4-chrome
+    - raptor-tp6-5-chrome
+    - raptor-tp6-6-chrome
     - raptor-speedometer-chrome
     - raptor-stylebench-chrome
     - raptor-motionmark-htmlsuite-chrome
     - raptor-motionmark-animometer-chrome
     - raptor-webaudio-chrome
     - raptor-sunspider-chrome
     - raptor-wasm-godot-chrome
 
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6-3.manifest
@@ -0,0 +1,9 @@
+[
+    {
+        "filename": "raptor-tp6-3.zip",
+        "size": 24512088,
+        "digest": "5b8080cf842a50fe2055127daa084fa9f133307965e4d8af246978150b8b3b11f1fb06cdf65ba69e13c875b77db14e7494cdb940376b264c3aefb5b53e22b892",
+        "algorithm": "sha512",
+        "unpack": true
+    }
+]
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6-4.manifest
@@ -0,0 +1,9 @@
+[
+    {
+        "filename": "raptor-tp6-4.zip",
+        "size": 1643188,
+        "digest": "0e9f2a23323f93f7a9839ab49d84555cce03bfab2196aa8670e604d6df390b22f0594527cecf7b2efd0449dd87507ec934bdcc74f1ebee68c22161f4104b6513",
+        "algorithm": "sha512",
+        "unpack": true
+    }
+]
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6-5.manifest
@@ -0,0 +1,9 @@
+[
+    {
+        "filename": "raptor-tp6-5.zip",
+        "size": 27670268,
+        "digest": "4a52cb6770062231f9f283ab42ac53634d91677146b4d91931f508de22bff262512775d0c93d80730287000907e438d467136768883691c3e20c1e6a8f475a03",
+        "algorithm": "sha512",
+        "unpack": true
+    }
+]
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6-6.manifest
@@ -0,0 +1,9 @@
+[
+    {
+        "filename": "raptor-tp6-6.zip",
+        "size": 2240008,
+        "digest": "21110ee297074413a72343bcb7d22ae88efce104de6e7eea5916e2dae004b29746ec6dd6155686cffed816e88d38ad1bbc04fe1253623ed8c0d4256a146d8b77",
+        "algorithm": "sha512",
+        "unpack": true
+    }
+]
\ No newline at end of file
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@@ -1,11 +1,15 @@
 # raptor pageload tests
 [include:tests/raptor-tp6-1.ini]
 [include:tests/raptor-tp6-2.ini]
+[include:tests/raptor-tp6-3.ini]
+[include:tests/raptor-tp6-4.ini]
+[include:tests/raptor-tp6-5.ini]
+[include:tests/raptor-tp6-6.ini]
 
 # raptor benchmark tests
 [include:tests/raptor-assorted-dom.ini]
 [include:tests/raptor-motionmark-animometer.ini]
 [include:tests/raptor-motionmark-htmlsuite.ini]
 [include:tests/raptor-speedometer.ini]
 [include:tests/raptor-stylebench.ini]
 [include:tests/raptor-sunspider.ini]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6-3.ini
@@ -0,0 +1,57 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-3
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-3.manifest
+page_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+# TTI/TTFI can take a while on some pages, and requires at least 5 seconds
+# beyond typical pageload time
+page_timeout = 30000
+gecko_profile_interval = 1
+gecko_profile_entries = 2000000
+
+[raptor-tp6-imdb-firefox]
+apps = firefox
+test_url = https://www.imdb.com/title/tt0084967/?ref_=nv_sr_2
+playback_recordings = imdb.mp
+measure = fnbpaint, dcf, ttfi
+
+[raptor-tp6-imgur-firefox]
+apps = firefox
+test_url = https://imgur.com/gallery/m5tYJL6
+playback_recordings = imgur.mp
+measure = fnbpaint, dcf, ttfi
+
+[raptor-tp6-wikia-firefox]
+apps = firefox
+test_url = http://fandom.wikia.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase
+playback_recordings = wikia.mp
+measure = fnbpaint, dcf, ttfi
+
+[raptor-tp6-imdb-chrome]
+apps = chrome
+test_url = https://www.imdb.com/title/tt0084967/?ref_=nv_sr_2
+playback_recordings = imdb.mp
+measure = fcp
+
+[raptor-tp6-imgur-chrome]
+apps = chrome
+test_url = https://imgur.com/gallery/m5tYJL6
+playback_recordings = imgur.mp
+measure = fcp
+
+[raptor-tp6-wikia-chrome]
+apps = chrome
+test_url = http://fandom.wikia.com/articles/fallout-76-will-live-and-die-on-the-creativity-of-its-playerbase
+playback_recordings = wikia.mp
+measure = fcp
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6-4.ini
@@ -0,0 +1,45 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-4
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-4.manifest
+page_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+# TTI/TTFI can take a while on some pages, and requires at least 5 seconds
+# beyond typical pageload time
+page_timeout = 30000
+gecko_profile_interval = 1
+gecko_profile_entries = 2000000
+
+[raptor-tp6-bing-firefox]
+apps = firefox
+test_url = https://www.bing.com/search?q=barack+obama
+playback_recordings = bing.mp
+measure = fnbpaint, dcf, ttfi
+
+[raptor-tp6-yandex-firefox]
+apps = firefox
+test_url = https://yandex.ru/search/?text=barack%20obama&lr=10115
+playback_recordings = yandex.mp
+measure = fnbpaint, dcf, ttfi
+
+[raptor-tp6-bing-chrome]
+apps = chrome
+test_url = https://www.bing.com/search?q=barack+obama
+playback_recordings = bing.mp
+measure = fcp
+
+[raptor-tp6-yandex-chrome]
+apps = chrome
+test_url = https://yandex.ru/search/?text=barack%20obama&lr=10115
+playback_recordings = yandex.mp
+measure = fcp
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6-5.ini
@@ -0,0 +1,46 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-5
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-5.manifest
+page_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+# TTI/TTFI can take a while on some pages, and requires at least 5 seconds
+# beyond typical pageload time
+page_timeout = 30000
+gecko_profile_interval = 1
+gecko_profile_entries = 2000000
+
+[raptor-tp6-apple-firefox]
+apps = firefox
+test_url = https://www.apple.com/macbook-pro/
+playback_recordings = apple.mp
+measure = fnbpaint, dcf, ttfi
+
+[raptor-tp6-microsoft-firefox]
+apps = firefox
+test_url = https://www.microsoft.com/en-us/windows/get-windows-10
+page_timeout = 60000
+playback_recordings = microsoft.mp
+measure = fnbpaint, dcf, ttfi
+
+[raptor-tp6-apple-chrome]
+apps = chrome
+test_url = https://www.apple.com/macbook-pro/
+playback_recordings = apple.mp
+measure = fcp
+
+[raptor-tp6-microsoft-chrome]
+apps = chrome
+test_url = https://www.microsoft.com/en-us/windows/get-windows-10
+playback_recordings = microsoft.mp
+measure = fcp
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6-6.ini
@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-6
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+python3_win_manifest = python3{x64}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-6.manifest
+page_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+# TTI/TTFI can take a while on some pages, and requires at least 5 seconds
+# beyond typical pageload time
+page_timeout = 30000
+gecko_profile_interval = 1
+gecko_profile_entries = 2000000
+
+[raptor-tp6-reddit-firefox]
+apps = firefox
+test_url = https://www.reddit.com/r/technology/comments/9sqwyh/we_posed_as_100_senators_to_run_ads_on_facebook/
+playback_recordings = reddit.mp
+measure = fnbpaint, dcf, ttfi
+
+[raptor-tp6-reddit-chrome]
+apps = chrome
+test_url = https://www.reddit.com/r/technology/comments/9sqwyh/we_posed_as_100_senators_to_run_ads_on_facebook/
+playback_recordings = reddit.mp
+measure = fcp
--- a/testing/raptor/webext/raptor/manifest.json
+++ b/testing/raptor/webext/raptor/manifest.json
@@ -8,20 +8,32 @@
   "name": "Raptor",
   "version": "0.1",
   "description": "Performance measurement framework prototype",
   "background": {
     "scripts": ["auto_gen_test_config.js", "runner.js"]
   },
   "content_scripts": [
     {
-      "matches": ["*://*.amazon.com/*",
+      "matches": [
+                  "*://*.apple.com/*",
+                  "*://*.amazon.com/*",
+                  "*://*.bing.com/*",
                   "*://*.facebook.com/*",
                   "*://*.google.com/*",
-                  "*://*.youtube.com/*"],
+                  "*://*.imdb.com/*",
+                  "*://*.imgur.com/*",
+                  "*://*.microsoft.com/*",
+                  "*://*.reddit.com/*",
+                  "*://*.vice.com/*",
+                  "*://*.wikia.com/*",
+                  "*://*.wikipedia.org/*",
+                  "*://*.youtube.com/*",
+                  "*://*.yandex.ru/*"
+                  ],
       "js": ["measure.js"]
     },
     {
       "matches": ["*://*/Speedometer/index.html*",
                   "*://*/StyleBench/*",
                   "*://*/MotionMark/*",
                   "*://*/SunSpider/*",
                   "*://*/webaudio/*",