Bug 1529212 Add new Raptor tests in tp6m-7 r=rwood
authoralexandru.ionescu <alexandru.ionescu@softvision.ro>
Mon, 11 Mar 2019 14:30:12 +0000
changeset 524381 e37e74e923b9bfb27f9330450e89395865fe5087
parent 524380 9c46b805faebe42a7abc0a84fe5f35bacbb8970a
child 524382 7aefe7888eabbdec1c7b013f0e6d2138f6d057f6
push id2032
push userffxbld-merge
push dateMon, 13 May 2019 09:36:57 +0000
treeherdermozilla-release@455c1065dcbe [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrwood
bugs1529212
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1529212 Add new Raptor tests in tp6m-7 r=rwood Differential Revision: https://phabricator.services.mozilla.com/D21000
taskcluster/ci/test/raptor.yml
taskcluster/ci/test/test-sets.yml
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-imdb.manifest
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-jianshu.manifest
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-microsoft-support.manifest
testing/raptor/raptor/raptor.ini
testing/raptor/raptor/tests/raptor-tp6m-7.ini
testing/raptor/webext/raptor/manifest.json
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -288,16 +288,17 @@ raptor-tp6m-2-geckoview:
             android-hw-g5.*: ['try', 'mozilla-central']
             default: ['try', 'trunk', 'mozilla-beta']
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-tp6m-2
             - --app=geckoview
             - --binary=org.mozilla.geckoview_example
+            - --activity=GeckoViewActivity
 
 raptor-tp6m-5-geckoview:
     description: "Raptor tp6m-5 on Geckoview"
     try-name: raptor-tp6m-5-geckoview
     treeherder-symbol: Rap(tp6m-5)
     run-on-projects:
         by-test-platform:
             android-hw-p2-8-0-arm7.*: ['try', 'mozilla-central']
@@ -307,16 +308,47 @@ raptor-tp6m-5-geckoview:
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-tp6m-5
             - --app=geckoview
             - --binary=org.mozilla.geckoview_example
             - --activity=GeckoViewActivity
 
+raptor-tp6m-6-geckoview:
+    description: "Raptor tp6m-6 on Geckoview"
+    try-name: raptor-tp6m-6-geckoview
+    treeherder-symbol: Rap(tp6m-6)
+    run-on-projects:
+        by-test-platform:
+            android-hw-p2-8-0-arm7.*: ['try', 'mozilla-central']
+            android-hw-g5.*: ['try', 'mozilla-central']
+            default: ['try', 'trunk', 'mozilla-beta']
+    target: geckoview_example.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-6
+            - --app=geckoview
+            - --binary=org.mozilla.geckoview_example
+
+raptor-tp6m-7-geckoview:
+    description: "Raptor tp6m-7 on Geckoview"
+    try-name: raptor-tp6m-7-geckoview
+    treeherder-symbol: Rap(tp6m-7)
+    run-on-projects: ['try', 'mozilla-central']
+    target: geckoview_example.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-7
+            - --app=geckoview
+            - --binary=org.mozilla.geckoview_example
+            - --activity=GeckoViewActivity
+
 raptor-tp6m-8-geckoview:
     description: "Raptor tp6m-8 on Geckoview"
     try-name: raptor-tp6m-8-geckoview
     treeherder-symbol: Rap(tp6m-8)
     run-on-projects:
         by-test-platform:
             android-hw-p2-8-0-arm7.*: ['try', 'mozilla-central']
             android-hw-g5.*: ['try', 'mozilla-central']
@@ -324,33 +356,16 @@ raptor-tp6m-8-geckoview:
     target: geckoview_example.apk
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-tp6m-8
             - --app=geckoview
             - --binary=org.mozilla.geckoview_example
 
-raptor-tp6m-6-geckoview:
-    description: "Raptor tp6m-6 on Geckoview"
-    try-name: raptor-tp6m-6-geckoview
-    treeherder-symbol: Rap(tp6m-6)
-    run-on-projects:
-        by-test-platform:
-            android-hw-p2-8-0-arm7.*: ['try', 'mozilla-central']
-            android-hw-g5.*: ['try', 'mozilla-central']
-            default: ['try', 'trunk', 'mozilla-beta']
-    target: geckoview_example.apk
-    tier: 2
-    mozharness:
-        extra-options:
-            - --test=raptor-tp6m-6
-            - --app=geckoview
-            - --binary=org.mozilla.geckoview_example
-
 raptor-speedometer-firefox:
     description: "Raptor Speedometer on Firefox"
     try-name: raptor-speedometer-firefox
     treeherder-symbol: Rap(sp)
     mozharness:
         extra-options:
             - --test=raptor-speedometer
 
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -410,24 +410,26 @@ android-hw-aarch64-opt-unittests:
     - jittest
 
 android-hw-arm7-raptor:
     - raptor-speedometer-geckoview
     - raptor-tp6m-1-geckoview
     - raptor-tp6m-2-geckoview
     - raptor-tp6m-5-geckoview
     - raptor-tp6m-6-geckoview
+    - raptor-tp6m-7-geckoview
     - raptor-tp6m-8-geckoview
 
 android-hw-aarch64-raptor:
     - raptor-speedometer-geckoview
     - raptor-tp6m-1-geckoview
     - raptor-tp6m-2-geckoview
     - raptor-tp6m-5-geckoview
     - raptor-tp6m-6-geckoview
+    - raptor-tp6m-7-geckoview
     - raptor-tp6m-8-geckoview
 
 android-hw-arm7-raptor-power:
     - raptor-speedometer-geckoview-power
 
 android-hw-aarch64-raptor-power:
     - raptor-speedometer-geckoview-power
 
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-imdb.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 6751756,
+    "visibility": "public",
+    "digest": "6f7d0bf3a13d2b082ba900b10781d0c434dc651001f73a78bc1aa18091fc2f9b6d7f0b114b46a845ee1832bf0b39703fa796f96992732066755aceddd2eda746",
+    "algorithm": "sha512",
+    "filename": "android-imdb.zip",
+    "unpack": true
+  }
+]
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-jianshu.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 6086637,
+    "visibility": "public",
+    "digest": "cebc9d79b1fb4023fa0a16c1cf133d11775aefa87c6fe5c6b630b56bd0715cd50414a3a948051ef2838b4c3bd5d7396b50408da572bf5c1a2fc69673131bb356",
+    "algorithm": "sha512",
+    "filename": "android-jianshu.zip",
+    "unpack": true
+  }
+]
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-microsoft-support.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 1641264,
+    "visibility": "public",
+    "digest": "f58436c869b629f52499bb3fc9b91ed776225d5cd623dc75ee3185af4ee1134ac67e1e4446567995625973d2b05b7f5533c08c145781e82befef978b962721ff",
+    "algorithm": "sha512",
+    "filename": "android-microsoft-support.zip",
+    "unpack": true
+  }
+]
\ No newline at end of file
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@@ -13,16 +13,17 @@
 # raptor pageload binast tests desktop
 [include:tests/raptor-tp6-binast-1.ini]
 
 # raptor pageload tests mobile
 [include:tests/raptor-tp6m-1.ini]
 [include:tests/raptor-tp6m-2.ini]
 [include:tests/raptor-tp6m-5.ini]
 [include:tests/raptor-tp6m-6.ini]
+[include:tests/raptor-tp6m-7.ini]
 [include:tests/raptor-tp6m-8.ini]
 
 # raptor benchmark tests
 [include:tests/raptor-assorted-dom.ini]
 [include:tests/raptor-motionmark-animometer.ini]
 [include:tests/raptor-motionmark-htmlsuite.ini]
 [include:tests/raptor-speedometer.ini]
 [include:tests/raptor-stylebench.ini]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-7.ini
@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-7
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+page_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+
+[raptor-tp6m-microsoft-support-geckoview]
+apps = geckoview
+test_url = https://support.microsoft.com/en-us
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-microsoft-support.manifest
+playback_recordings = android-microsoft-support.mp
+measure = fnbpaint, fcp, dcf, ttfi, loadtime
+disabled = Bug 1533306 Investigating tp6m-7 raptor test raptor-tp6m-microsoft-support-geckoview
+
+[raptor-tp6m-jianshu-geckoview]
+apps = geckoview
+test_url = https://www.jianshu.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-jianshu.manifest
+playback_recordings = android-jianshu.mp
+measure = fnbpaint, fcp, dcf, ttfi, loadtime
+
+[raptor-tp6m-imdb-geckoview]
+apps = geckoview
+test_url = https://m.imdb.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-imdb.manifest
+playback_recordings = android-imdb.mp
+measure = fnbpaint, fcp, dcf, ttfi, loadtime
--- a/testing/raptor/webext/raptor/manifest.json
+++ b/testing/raptor/webext/raptor/manifest.json
@@ -22,16 +22,17 @@
                   "*://*.ebay.com/*",
                   "*://*.ebay-kleinanzeigen.de/*",
                   "*://*.espn.com/*",
                   "*://*.facebook.com/*",
                   "*://*.google.com/*",
                   "*://*.imdb.com/*",
                   "*://*.imgur.com/*",
                   "*://*.instagram.com/*",
+                  "*://*.jianshu.com/*",
                   "*://*.microsoft.com/*",
                   "*://*.paypal.com/*",
                   "*://*.pinterest.com/*",
                   "*://*.reddit.com/*",
                   "*://*.stackoverflow.com/*",
                   "*://*.tumblr.com/*",
                   "*://*.twitter.com/*",
                   "*://*.vice.com/*",