Bug 1522972 - Raptor cold-load tests on Firefox and Chromium desktop; #perftest r=davehunt
authorRob Wood <rwood@mozilla.com>
Wed, 01 May 2019 14:52:59 +0000
changeset 472154 918567a07dbb60dafc2b971afc4ce1610992d069
parent 472153 984c175f86f98a22d6a56e40773f5e3f773a004a
child 472155 83eece97230e06f4b83940dc862a35e20c836532
push id35949
push usernerli@mozilla.com
push dateThu, 02 May 2019 04:38:13 +0000
treeherdermozilla-central@09753a1a153d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdavehunt
bugs1522972
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1522972 - Raptor cold-load tests on Firefox and Chromium desktop; #perftest r=davehunt Differential Revision: https://phabricator.services.mozilla.com/D29032
taskcluster/ci/test/raptor-chromium.yml
taskcluster/ci/test/raptor.yml
taskcluster/ci/test/test-sets.yml
testing/raptor/raptor/raptor.ini
testing/raptor/raptor/raptor.py
testing/raptor/raptor/tests/raptor-tp6-cold-1.ini
testing/raptor/raptor/tests/raptor-tp6-cold-2.ini
testing/raptor/raptor/tests/raptor-tp6-cold-3.ini
testing/raptor/raptor/tests/raptor-tp6-cold-4.ini
--- a/taskcluster/ci/test/raptor-chromium.yml
+++ b/taskcluster/ci/test/raptor-chromium.yml
@@ -267,8 +267,44 @@ raptor-wasm-godot-chromium:
     description: "Raptor Wasm Godot on Chromium"
     try-name: raptor-wasm-godot-chromium
     treeherder-symbol: Rap-Cr(godot)
     max-run-time: 1500
     mozharness:
         extra-options:
             - --test=raptor-wasm-godot
             - --app=chromium
+
+raptor-tp6-1-chromium-cold:
+    description: "Raptor tp6-1 cold page-load on Chromium"
+    try-name: raptor-tp6-1-chromium-cold
+    treeherder-symbol: Rap-Cr(tp6-c-1)
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-cold-1
+            - --app=chromium
+
+raptor-tp6-2-chromium-cold:
+    description: "Raptor tp6-2 cold page-load on Chromium"
+    try-name: raptor-tp6-2-chromium-cold
+    treeherder-symbol: Rap-Cr(tp6-c-2)
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-cold-2
+            - --app=chromium
+
+raptor-tp6-3-chromium-cold:
+    description: "Raptor tp6-3 cold page-load on Chromium"
+    try-name: raptor-tp6-3-chromium-cold
+    treeherder-symbol: Rap-Cr(tp6-c-3)
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-cold-3
+            - --app=chromium
+
+raptor-tp6-4-chromium-cold:
+    description: "Raptor tp6-4 cold page-load on Chromium"
+    try-name: raptor-tp6-4-chromium-cold
+    treeherder-symbol: Rap-Cr(tp6-c-4)
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-cold-4
+            - --app=chromium
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -1466,8 +1466,44 @@ raptor-wasm-godot-ion-firefox-profiling:
     treeherder-symbol: Rap-Prof(godot-i)
     run-on-projects: ['mozilla-central', 'try']
     max-run-time: 900
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-wasm-godot-ion
             - --gecko-profile
+
+raptor-tp6-1-firefox-cold:
+    description: "Raptor tp6-1 cold page-load on Firefox"
+    try-name: raptor-tp6-1-firefox-cold
+    treeherder-symbol: Rap(tp6-c-1)
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-cold-1
+
+raptor-tp6-2-firefox-cold:
+    description: "Raptor tp6-2 cold page-load on Firefox"
+    try-name: raptor-tp6-2-firefox-cold
+    treeherder-symbol: Rap(tp6-c-2)
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-cold-2
+
+raptor-tp6-3-firefox-cold:
+    description: "Raptor tp6-3 cold page-load on Firefox"
+    try-name: raptor-tp6-3-firefox-cold
+    treeherder-symbol: Rap(tp6-c-3)
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-cold-3
+
+raptor-tp6-4-firefox-cold:
+    description: "Raptor tp6-4 cold page-load on Firefox"
+    try-name: raptor-tp6-4-firefox-cold
+    treeherder-symbol: Rap(tp6-c-4)
+    tier: 2
+    mozharness:
+        extra-options:
+            - --test=raptor-tp6-cold-4
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -91,16 +91,20 @@ raptor-firefox:
     - raptor-tp6-binast-1-firefox
     - raptor-speedometer-firefox
     - raptor-stylebench-firefox
     - raptor-motionmark-htmlsuite-firefox
     - raptor-motionmark-animometer-firefox
     - raptor-webaudio-firefox
     - raptor-sunspider-firefox
     - raptor-wasm-godot-firefox
+    - raptor-tp6-1-firefox-cold
+    - raptor-tp6-2-firefox-cold
+    - raptor-tp6-3-firefox-cold
+    - raptor-tp6-4-firefox-cold
 
 raptor-profiling:
     - raptor-tp6-1-firefox-profiling
     - raptor-tp6-2-firefox-profiling
     - raptor-tp6-3-firefox-profiling
     - raptor-tp6-4-firefox-profiling
     - raptor-tp6-5-firefox-profiling
     - raptor-tp6-6-firefox-profiling
@@ -129,16 +133,20 @@ raptor-chromium:
     - raptor-tp6-10-chromium
     - raptor-speedometer-chromium
     - raptor-stylebench-chromium
     - raptor-motionmark-htmlsuite-chromium
     - raptor-motionmark-animometer-chromium
     - raptor-webaudio-chromium
     - raptor-sunspider-chromium
     - raptor-wasm-godot-chromium
+    - raptor-tp6-1-chromium-cold
+    - raptor-tp6-2-chromium-cold
+    - raptor-tp6-3-chromium-cold
+    - raptor-tp6-4-chromium-cold
 
 # Fetch tasks are only supported on Linux for now,
 # so these need to be separate sets.
 raptor-fetch-firefox:
     - raptor-unity-webgl-firefox
     - raptor-wasm-misc-firefox
     - raptor-wasm-misc-baseline-firefox
     - raptor-wasm-misc-ion-firefox
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@@ -1,20 +1,26 @@
-# raptor pageload tests desktop
+# raptor warm pageload tests desktop
 [include:tests/raptor-tp6-1.ini]
 [include:tests/raptor-tp6-2.ini]
 [include:tests/raptor-tp6-3.ini]
 [include:tests/raptor-tp6-4.ini]
 [include:tests/raptor-tp6-5.ini]
 [include:tests/raptor-tp6-6.ini]
 [include:tests/raptor-tp6-7.ini]
 [include:tests/raptor-tp6-8.ini]
 [include:tests/raptor-tp6-9.ini]
 [include:tests/raptor-tp6-10.ini]
 
+# raptor cold pageload tests desktop
+[include:tests/raptor-tp6-cold-1.ini]
+[include:tests/raptor-tp6-cold-2.ini]
+[include:tests/raptor-tp6-cold-3.ini]
+[include:tests/raptor-tp6-cold-4.ini]
+
 # raptor pageload binast tests desktop
 [include:tests/raptor-tp6-binast-1.ini]
 
 # raptor warm pageload tests mobile
 [include:tests/raptor-tp6m-1.ini]
 [include:tests/raptor-tp6m-2.ini]
 [include:tests/raptor-tp6m-3.ini]
 [include:tests/raptor-tp6m-4.ini]
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -431,16 +431,79 @@ class RaptorDesktop(Raptor):
         self.runner.start()
         proc = self.runner.process_handler
         self.output_handler.proc = proc
 
         # give our control server the browser process so it can shut it down later
         self.control_server.browser_proc = proc
 
     def run_test(self, test, timeout=None):
+        # tests will be run warm (i.e. NO browser restart between page-cycles)
+        # unless otheriwse specified in the test INI by using 'cold = true'
+        if test.get('cold', False) is True:
+            self.run_test_cold(test, timeout)
+        else:
+            self.run_test_warm(test, timeout)
+
+    def run_test_cold(self, test, timeout=None):
+        '''
+        Run the Raptor test but restart the entire browser app between page-cycles.
+
+        Note: For page-load tests, playback will only be started once - at the beginning of all
+        browser cycles, and then stopped after all cycles are finished. That includes the import
+        of the mozproxy ssl cert and turning on the browser proxy.
+
+        Since we're running in cold-mode, before this point (in manifest.py) the
+        'expected-browser-cycles' value was already set to the initial 'page-cycles' value;
+        and the 'page-cycles' value was set to 1 as we want to perform one page-cycle per
+        browser restart.
+
+        The 'browser-cycle' value is the current overall browser start iteration. The control
+        server will receive the current 'browser-cycle' and the 'expected-browser-cycles' in
+        each results set received; and will pass that on as part of the results so that the
+        results processing will know results for multiple browser cycles are being received.
+
+        The default will be to run in warm mode; unless 'cold = true' is set in the test INI.
+        '''
+        self.log.info("test %s is running in cold mode; browser WILL be restarted between "
+                      "page cycles" % test['name'])
+
+        for test['browser_cycle'] in range(1, test['expected_browser_cycles'] + 1):
+
+            self.log.info("begin browser cycle %d of %d for test %s"
+                          % (test['browser_cycle'], test['expected_browser_cycles'], test['name']))
+
+            self.run_test_setup(test)
+
+            if test['browser_cycle'] == 1:
+
+                if test.get('playback') is not None:
+                    self.start_playback(test)
+
+                if self.config['host'] not in ('localhost', '127.0.0.1'):
+                    self.delete_proxy_settings_from_profile()
+
+            else:
+                # initial browser profile was already created before run_test was called;
+                # now additional browser cycles we want to create a new one each time
+                self.create_browser_profile()
+
+                self.run_test_setup(test)
+
+            # now start the browser/app under test
+            self.launch_desktop_browser(test)
+
+            # set our control server flag to indicate we are running the browser/app
+            self.control_server._finished = False
+
+            self.wait_for_test_finish(test, timeout)
+
+        self.run_test_teardown()
+
+    def run_test_warm(self, test, timeout=None):
         self.run_test_setup(test)
 
         try:
             if test.get('playback') is not None:
                 self.start_playback(test)
 
             if self.config['host'] not in ('localhost', '127.0.0.1'):
                 self.delete_proxy_settings_from_profile()
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6-cold-1.ini
@@ -0,0 +1,39 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-cold-1
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+playback_version = 4.0.4
+playback_binary_manifest = mitmproxy-rel-bin-4.0.4-{platform}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-1.manifest
+browser_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+gecko_profile_interval = 1
+gecko_profile_entries = 14000000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6-amazon-firefox-cold]
+apps = firefox
+test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
+playback_recordings = amazon.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6-amazon-chromium-cold]
+apps = chromium
+test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
+playback_recordings = amazon.mp
+measure = fcp, loadtime
+
+[raptor-tp6-amazon-chrome-cold]
+apps = chrome
+test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
+playback_recordings = amazon.mp
+measure = fcp, loadtime
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6-cold-2.ini
@@ -0,0 +1,39 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-cold-2
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+playback_version = 4.0.4
+playback_binary_manifest = mitmproxy-rel-bin-4.0.4-{platform}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-1.manifest
+browser_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+gecko_profile_interval = 1
+gecko_profile_entries = 14000000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6-facebook-firefox-cold]
+apps = firefox
+test_url = https://www.facebook.com
+playback_recordings = facebook.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6-facebook-chromium-cold]
+apps = chromium
+test_url = https://www.facebook.com
+playback_recordings = facebook.mp
+measure = fcp, loadtime
+
+[raptor-tp6-facebook-chrome-cold]
+apps = chrome
+test_url = https://www.facebook.com
+playback_recordings = facebook.mp
+measure = fcp, loadtime
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6-cold-3.ini
@@ -0,0 +1,42 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-cold-3
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+playback_version = 4.0.4
+playback_binary_manifest = mitmproxy-rel-bin-4.0.4-{platform}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-1.manifest
+browser_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+gecko_profile_interval = 1
+gecko_profile_entries = 14000000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6-google-firefox-cold]
+apps = firefox
+# note: use the full url as the first part (without '&cad=h') redirects
+# to the url + '&cad=h'; that redirection causes measure.js content
+# to be loaded into that page also; resulting in 2 fnbpaint values etc.
+test_url = https://www.google.com/search?hl=en&q=barack+obama&cad=h
+playback_recordings = google-search.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6-google-chromium-cold]
+apps = chromium
+test_url = https://www.google.com/#hl=en&q=barack+obama
+playback_recordings = google-search.mp
+measure = fcp, loadtime
+
+[raptor-tp6-google-chrome-cold]
+apps = chrome
+test_url = https://www.google.com/#hl=en&q=barack+obama
+playback_recordings = google-search.mp
+measure = fcp, loadtime
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-tp6-cold-4.ini
@@ -0,0 +1,39 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# raptor tp6-cold-4
+
+[DEFAULT]
+type =  pageload
+playback = mitmproxy
+playback_version = 4.0.4
+playback_binary_manifest = mitmproxy-rel-bin-4.0.4-{platform}.manifest
+playback_pageset_manifest = mitmproxy-recordings-raptor-tp6-1.manifest
+browser_cycles = 25
+unit = ms
+lower_is_better = true
+alert_threshold = 2.0
+page_timeout = 60000
+gecko_profile_interval = 1
+gecko_profile_entries = 14000000
+alert_on = fcp, loadtime
+cold = true
+
+[raptor-tp6-youtube-firefox-cold]
+apps = firefox
+test_url = https://www.youtube.com
+playback_recordings = youtube.mp
+measure = fnbpaint, fcp, dcf, loadtime
+
+[raptor-tp6-youtube-chromium-cold]
+apps = chromium
+test_url = https://www.youtube.com
+playback_recordings = youtube.mp
+measure = fcp, loadtime
+
+[raptor-tp6-youtube-chrome-cold]
+apps = chrome
+test_url = https://www.youtube.com
+playback_recordings = youtube.mp
+measure = fcp, loadtime