Bug 1529211 Add new Raptor tests in tp6m-6 r=davehunt
authoralexandru.ionescu <alexandru.ionescu@softvision.ro>
Thu, 07 Mar 2019 16:37:30 +0000
changeset 463125 ef2b461b0a90a509075824a47115a7c94a3eb25b
parent 463124 5734a9a9153133cd366f75742cc958ce205a8b5d
child 463126 3c21d47f9ddf9853519f2f7c49a1fe1ca5aa6463
push id35668
push userbtara@mozilla.com
push dateFri, 08 Mar 2019 21:50:53 +0000
treeherdermozilla-central@14778fd00dc5 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdavehunt
bugs1529211
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1529211 Add new Raptor tests in tp6m-6 r=davehunt Differential Revision: https://phabricator.services.mozilla.com/D20820
taskcluster/ci/test/raptor.yml
taskcluster/ci/test/test-sets.yml
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-bbc.manifest
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-reddit.manifest
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-stackoverflow.manifest
testing/raptor/raptor/raptor.ini
testing/raptor/raptor/tests/raptor-tp6m-6.ini
testing/raptor/webext/raptor/manifest.json
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -324,16 +324,33 @@ raptor-tp6m-8-geckoview:
     target: geckoview_example.apk
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-tp6m-8
             - --app=geckoview
             - --binary=org.mozilla.geckoview_example
 
+raptor-tp6m-6-geckoview:
+    description: "Raptor tp6m-6 on Geckoview"
+    try-name: raptor-tp6m-6-geckoview
+    treeherder-symbol: Rap(tp6m-6)
+    run-on-projects:
+        by-test-platform:
+            android-hw-p2-8-0-arm7.*: ['try', 'mozilla-central']
+            android-hw-g5.*: ['try', 'mozilla-central']
+            default: ['try', 'trunk', 'mozilla-beta']
+    target: geckoview_example.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-6
+            - --app=geckoview
+            - --binary=org.mozilla.geckoview_example
+
 raptor-speedometer-firefox:
     description: "Raptor Speedometer on Firefox"
     try-name: raptor-speedometer-firefox
     treeherder-symbol: Rap(sp)
     mozharness:
         extra-options:
             - --test=raptor-speedometer
 
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -409,23 +409,25 @@ android-hw-arm7-debug-unittests:
 android-hw-aarch64-opt-unittests:
     - jittest
 
 android-hw-arm7-raptor:
     - raptor-speedometer-geckoview
     - raptor-tp6m-1-geckoview
     - raptor-tp6m-2-geckoview
     - raptor-tp6m-5-geckoview
+    - raptor-tp6m-6-geckoview
     - raptor-tp6m-8-geckoview
 
 android-hw-aarch64-raptor:
     - raptor-speedometer-geckoview
     - raptor-tp6m-1-geckoview
     - raptor-tp6m-2-geckoview
     - raptor-tp6m-5-geckoview
+    - raptor-tp6m-6-geckoview
     - raptor-tp6m-8-geckoview
 
 android-hw-arm7-raptor-power:
     - raptor-speedometer-geckoview-power
 
 android-hw-aarch64-raptor-power:
     - raptor-speedometer-geckoview-power
 
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-bbc.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 2047073,
+    "visibility": "public",
+    "digest": "abdfd7581b34d8af436291e0c22c36f33f2d9ff9efbd49854312d7dce3aab20895040715f00bb8bb7f92efe411795aae73d960d8c37c35810bbb6bc538da1f9f",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-recordings-raptor-tp6m-bbc.zip",
+    "unpack": true
+  }
+]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-reddit.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 5310402,
+    "visibility": "public",
+    "digest": "c6e4cc7e9c7e600e65a56a690201d7a9f6e803c42f537d2691f038ab39a0b95c37ff33d84f0f981b5f9943b4041203b8c2493b12e37f72e4dfd439074d17dbcc",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-recordings-raptor-tp6m-reddit.zip",
+    "unpack": true
+  }
+]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-stackoverflow.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 720295,
+    "visibility": "public",
+    "digest": "2ddbe6e68df1a78ce5983a70ba47f4125afb6209cfceedae964a4a86dc68d30480e7c10493e77b55aefc0c4a5e79706f6126c6684d29d5f41aaffc0463944009",
+    "algorithm": "sha512",
+    "filename": "mitmproxy-recordings-raptor-tp6m-stackoverflow.zip",
+    "unpack": true
+  }
+]
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@@ -12,16 +12,17 @@
 
 # raptor pageload binast tests desktop
 [include:tests/raptor-tp6-binast-1.ini]
 
 # raptor pageload tests mobile
 [include:tests/raptor-tp6m-1.ini]
 [include:tests/raptor-tp6m-2.ini]
 [include:tests/raptor-tp6m-5.ini]
+[include:tests/raptor-tp6m-6.ini]
 [include:tests/raptor-tp6m-8.ini]
 
 # raptor benchmark tests
 [include:tests/raptor-assorted-dom.ini]
 [include:tests/raptor-motionmark-animometer.ini]
 [include:tests/raptor-motionmark-htmlsuite.ini]
 [include:tests/raptor-speedometer.ini]
 [include:tests/raptor-stylebench.ini]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-6.ini
@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-6
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+page_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+
+[raptor-tp6m-reddit-geckoview]
+apps = geckoview
+test_url = https://www.reddit.com
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-reddit.manifest
+playback_recordings = mitmproxy-recordings-raptor-tp6m-reddit.mp
+measure = fnbpaint, fcp, dcf, ttfi, loadtime
+
+[raptor-tp6m-bbc-geckoview]
+apps = geckoview
+test_url = https://www.bbc.com/news/business-47245877
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-bbc.manifest
+playback_recordings = mitmproxy-recordings-raptor-tp6m-bbc.mp
+measure = fnbpaint, fcp, dcf, ttfi, loadtime
+disabled = Bug 1533304 Investigating tp6m-6 raptor test for raptor-tp6m-bbc-geckoview
+
+[raptor-tp6m-stackoverflow-geckoview]
+apps = geckoview
+test_url = https://stackoverflow.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-stackoverflow.manifest
+playback_recordings = mitmproxy-recordings-raptor-tp6m-stackoverflow.mp
+measure = fnbpaint, fcp, dcf, ttfi, loadtime
--- a/testing/raptor/webext/raptor/manifest.json
+++ b/testing/raptor/webext/raptor/manifest.json
@@ -12,29 +12,31 @@
     "scripts": ["auto_gen_test_config.js", "runner.js"]
   },
   "content_scripts": [
     {
       "matches": [
                   "*://*.allrecipes.com/*",
                   "*://*.apple.com/*",
                   "*://*.amazon.com/*",
+                  "*://*.bbc.com/*",
                   "*://*.bing.com/*",
                   "*://*.ebay.com/*",
                   "*://*.ebay-kleinanzeigen.de/*",
                   "*://*.espn.com/*",
                   "*://*.facebook.com/*",
                   "*://*.google.com/*",
                   "*://*.imdb.com/*",
                   "*://*.imgur.com/*",
                   "*://*.instagram.com/*",
                   "*://*.microsoft.com/*",
                   "*://*.paypal.com/*",
                   "*://*.pinterest.com/*",
                   "*://*.reddit.com/*",
+                  "*://*.stackoverflow.com/*",
                   "*://*.tumblr.com/*",
                   "*://*.twitter.com/*",
                   "*://*.vice.com/*",
                   "*://*.web.de/*",
                   "*://*.wikia.com/*",
                   "*://*.wikipedia.org/*",
                   "*://*.yahoo.com/*",
                   "*://*.youtube.com/*",