Bug 1529212 Add new Raptor tests in tp6m-7 r=rwood
authoralexandru.ionescu <alexandru.ionescu@softvision.ro>
Mon, 11 Mar 2019 14:30:12 +0000
changeset 521383 e37e74e923b9
parent 521382 9c46b805faeb
child 521384 7aefe7888eab
push id10866
push usernerli@mozilla.com
push dateTue, 12 Mar 2019 18:59:09 +0000
treeherdermozilla-beta@445c24a51727 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrwood
bugs1529212
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1529212 Add new Raptor tests in tp6m-7 r=rwood Differential Revision: https://phabricator.services.mozilla.com/D21000
taskcluster/ci/test/raptor.yml
taskcluster/ci/test/test-sets.yml
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-imdb.manifest
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-jianshu.manifest
testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-microsoft-support.manifest
testing/raptor/raptor/raptor.ini
testing/raptor/raptor/tests/raptor-tp6m-7.ini
testing/raptor/webext/raptor/manifest.json
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -288,16 +288,17 @@ raptor-tp6m-2-geckoview:
             android-hw-g5.*: ['try', 'mozilla-central']
             default: ['try', 'trunk', 'mozilla-beta']
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-tp6m-2
             - --app=geckoview
             - --binary=org.mozilla.geckoview_example
+            - --activity=GeckoViewActivity
 
 raptor-tp6m-5-geckoview:
     description: "Raptor tp6m-5 on Geckoview"
     try-name: raptor-tp6m-5-geckoview
     treeherder-symbol: Rap(tp6m-5)
     run-on-projects:
         by-test-platform:
             android-hw-p2-8-0-arm7.*: ['try', 'mozilla-central']
@@ -307,16 +308,47 @@ raptor-tp6m-5-geckoview:
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-tp6m-5
             - --app=geckoview
             - --binary=org.mozilla.geckoview_example
             - --activity=GeckoViewActivity
 
+raptor-tp6m-6-geckoview:
+    description: "Raptor tp6m-6 on Geckoview"
+    try-name: raptor-tp6m-6-geckoview
+    treeherder-symbol: Rap(tp6m-6)
+    run-on-projects:
+        by-test-platform:
+            android-hw-p2-8-0-arm7.*: ['try', 'mozilla-central']
+            android-hw-g5.*: ['try', 'mozilla-central']
+            default: ['try', 'trunk', 'mozilla-beta']
+    target: geckoview_example.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-6
+            - --app=geckoview
+            - --binary=org.mozilla.geckoview_example
+
+raptor-tp6m-7-geckoview:
+    description: "Raptor tp6m-7 on Geckoview"
+    try-name: raptor-tp6m-7-geckoview
+    treeherder-symbol: Rap(tp6m-7)
+    run-on-projects: ['try', 'mozilla-central']
+    target: geckoview_example.apk
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6m-7
+            - --app=geckoview
+            - --binary=org.mozilla.geckoview_example
+            - --activity=GeckoViewActivity
+
 raptor-tp6m-8-geckoview:
     description: "Raptor tp6m-8 on Geckoview"
     try-name: raptor-tp6m-8-geckoview
     treeherder-symbol: Rap(tp6m-8)
     run-on-projects:
         by-test-platform:
             android-hw-p2-8-0-arm7.*: ['try', 'mozilla-central']
             android-hw-g5.*: ['try', 'mozilla-central']
@@ -324,33 +356,16 @@ raptor-tp6m-8-geckoview:
     target: geckoview_example.apk
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-tp6m-8
             - --app=geckoview
             - --binary=org.mozilla.geckoview_example
 
-raptor-tp6m-6-geckoview:
-    description: "Raptor tp6m-6 on Geckoview"
-    try-name: raptor-tp6m-6-geckoview
-    treeherder-symbol: Rap(tp6m-6)
-    run-on-projects:
-        by-test-platform:
-            android-hw-p2-8-0-arm7.*: ['try', 'mozilla-central']
-            android-hw-g5.*: ['try', 'mozilla-central']
-            default: ['try', 'trunk', 'mozilla-beta']
-    target: geckoview_example.apk
-    tier: 2
-    mozharness:
-        extra-options:
-            - --test=raptor-tp6m-6
-            - --app=geckoview
-            - --binary=org.mozilla.geckoview_example
-
 raptor-speedometer-firefox:
     description: "Raptor Speedometer on Firefox"
     try-name: raptor-speedometer-firefox
     treeherder-symbol: Rap(sp)
     mozharness:
         extra-options:
             - --test=raptor-speedometer
 
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -410,24 +410,26 @@ android-hw-aarch64-opt-unittests:
     - jittest
 
 android-hw-arm7-raptor:
     - raptor-speedometer-geckoview
     - raptor-tp6m-1-geckoview
     - raptor-tp6m-2-geckoview
     - raptor-tp6m-5-geckoview
     - raptor-tp6m-6-geckoview
+    - raptor-tp6m-7-geckoview
     - raptor-tp6m-8-geckoview
 
 android-hw-aarch64-raptor:
     - raptor-speedometer-geckoview
     - raptor-tp6m-1-geckoview
     - raptor-tp6m-2-geckoview
     - raptor-tp6m-5-geckoview
     - raptor-tp6m-6-geckoview
+    - raptor-tp6m-7-geckoview
     - raptor-tp6m-8-geckoview
 
 android-hw-arm7-raptor-power:
     - raptor-speedometer-geckoview-power
 
 android-hw-aarch64-raptor-power:
     - raptor-speedometer-geckoview-power
 
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-imdb.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 6751756,
+    "visibility": "public",
+    "digest": "6f7d0bf3a13d2b082ba900b10781d0c434dc651001f73a78bc1aa18091fc2f9b6d7f0b114b46a845ee1832bf0b39703fa796f96992732066755aceddd2eda746",
+    "algorithm": "sha512",
+    "filename": "android-imdb.zip",
+    "unpack": true
+  }
+]
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-jianshu.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 6086637,
+    "visibility": "public",
+    "digest": "cebc9d79b1fb4023fa0a16c1cf133d11775aefa87c6fe5c6b630b56bd0715cd50414a3a948051ef2838b4c3bd5d7396b50408da572bf5c1a2fc69673131bb356",
+    "algorithm": "sha512",
+    "filename": "android-jianshu.zip",
+    "unpack": true
+  }
+]
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/mitmproxy-recordings-raptor-tp6m-microsoft-support.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 1641264,
+    "visibility": "public",
+    "digest": "f58436c869b629f52499bb3fc9b91ed776225d5cd623dc75ee3185af4ee1134ac67e1e4446567995625973d2b05b7f5533c08c145781e82befef978b962721ff",
+    "algorithm": "sha512",
+    "filename": "android-microsoft-support.zip",
+    "unpack": true
+  }
+]
\ No newline at end of file
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@@ -13,16 +13,17 @@
 # raptor pageload binast tests desktop
 [include:tests/raptor-tp6-binast-1.ini]
 
 # raptor pageload tests mobile
 [include:tests/raptor-tp6m-1.ini]
 [include:tests/raptor-tp6m-2.ini]
 [include:tests/raptor-tp6m-5.ini]
 [include:tests/raptor-tp6m-6.ini]
+[include:tests/raptor-tp6m-7.ini]
 [include:tests/raptor-tp6m-8.ini]
 
 # raptor benchmark tests
 [include:tests/raptor-assorted-dom.ini]
 [include:tests/raptor-motionmark-animometer.ini]
 [include:tests/raptor-motionmark-htmlsuite.ini]
 [include:tests/raptor-speedometer.ini]
 [include:tests/raptor-stylebench.ini]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6m-7.ini
@@ -0,0 +1,38 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6m-7
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy-android
+playback_binary_manifest = mitmproxy-rel-bin-{platform}.manifest
+page_cycles = 15
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+alert_on = fcp, loadtime
+
+[raptor-tp6m-microsoft-support-geckoview]
+apps = geckoview
+test_url = https://support.microsoft.com/en-us
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-microsoft-support.manifest
+playback_recordings = android-microsoft-support.mp
+measure = fnbpaint, fcp, dcf, ttfi, loadtime
+disabled = Bug 1533306 Investigating tp6m-7 raptor test raptor-tp6m-microsoft-support-geckoview
+
+[raptor-tp6m-jianshu-geckoview]
+apps = geckoview
+test_url = https://www.jianshu.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-jianshu.manifest
+playback_recordings = android-jianshu.mp
+measure = fnbpaint, fcp, dcf, ttfi, loadtime
+
+[raptor-tp6m-imdb-geckoview]
+apps = geckoview
+test_url = https://m.imdb.com/
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6m-imdb.manifest
+playback_recordings = android-imdb.mp
+measure = fnbpaint, fcp, dcf, ttfi, loadtime
--- a/testing/raptor/webext/raptor/manifest.json
+++ b/testing/raptor/webext/raptor/manifest.json
@@ -22,16 +22,17 @@
                   "*://*.ebay.com/*",
                   "*://*.ebay-kleinanzeigen.de/*",
                   "*://*.espn.com/*",
                   "*://*.facebook.com/*",
                   "*://*.google.com/*",
                   "*://*.imdb.com/*",
                   "*://*.imgur.com/*",
                   "*://*.instagram.com/*",
+                  "*://*.jianshu.com/*",
                   "*://*.microsoft.com/*",
                   "*://*.paypal.com/*",
                   "*://*.pinterest.com/*",
                   "*://*.reddit.com/*",
                   "*://*.stackoverflow.com/*",
                   "*://*.tumblr.com/*",
                   "*://*.twitter.com/*",
                   "*://*.vice.com/*",