Bug 1575575 - Add fullscreen and windowed youtube-playback power tests. r=perftest-reviewers,stephendonner,dhouse,rwood
authorGregory Mierzwinski <gmierz2@outlook.com>
Thu, 26 Sep 2019 22:18:54 +0000
changeset 495215 cfc6ee518965970faed1a5a8351b903ffd489ea2
parent 495214 66ded616f2a1d5c549ecbe748e2b866bb9bb180d
child 495216 1ba2f46eab88480a9df98498135ab9fece02f893
push id96528
push usergmierz2@outlook.com
push dateThu, 26 Sep 2019 22:39:41 +0000
treeherderautoland@cfc6ee518965 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest-reviewers, stephendonner, dhouse, rwood
bugs1575575
milestone71.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1575575 - Add fullscreen and windowed youtube-playback power tests. r=perftest-reviewers,stephendonner,dhouse,rwood This patch adds fullscreen and windowed youtube tests for the V9 and H264 encoding at 1080p30 and 1080p60. Each subtest runs for 20 page cycles which amounts to about 5 minutes each. It also begins adding these to power test tasks running on the macosx-1014 reference hardware. Differential Revision: https://phabricator.services.mozilla.com/D45067
taskcluster/ci/config.yml
taskcluster/ci/test/raptor.yml
taskcluster/ci/test/test-platforms.yml
taskcluster/ci/test/test-sets.yml
taskcluster/taskgraph/transforms/tests.py
taskcluster/taskgraph/util/workertypes.py
testing/mozbase/mozpower/mozpower/intel_power_gadget.py
testing/mozbase/mozpower/tests/test_intelpowergadget.py
testing/profiles/raptor/user.js
testing/raptor/raptor/raptor.ini
testing/raptor/raptor/raptor.py
testing/raptor/raptor/tests/benchmarks/raptor-youtube-playback-h264.ini
testing/raptor/raptor/tests/benchmarks/raptor-youtube-playback-v9.ini
--- a/taskcluster/ci/config.yml
+++ b/taskcluster/ci/config.yml
@@ -371,16 +371,21 @@ workers:
             implementation: generic-worker
             os: macosx
             worker-type: 'gecko-{alias}'
         t-osx-1014:
             provisioner: releng-hardware
             implementation: generic-worker
             os: macosx
             worker-type: 'gecko-{alias}'
+        t-osx-1014-power:
+            provisioner: releng-hardware
+            implementation: generic-worker
+            os: macosx
+            worker-type: 'gecko-{alias}'
         t-linux-xlarge-pgo:
             provisioner: aws-provisioner-v1
             implementation: docker-worker
             os: linux
             worker-type:
                 by-level:
                     '3': 'gecko-{level}-t-linux-xlarge'
                     default: 'gecko-t-linux-xlarge'
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -654,16 +654,40 @@ raptor-youtube-playback-firefox:
         by-test-platform:
             windows10-aarch64/opt: 3600
             default: 2700
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-youtube-playback
 
+raptor-youtube-playback-v9-power-firefox:
+    description: "Raptor V9 YouTube Playback on Firefox with Power Testing"
+    try-name: raptor-youtube-playback-v9-power-firefox
+    treeherder-symbol: Rap(ytp-v9-p)
+    run-on-projects: ['mozilla-central']
+    max-run-time: 2700
+    tier: 3
+    mozharness:
+        extra-options:
+            - --test=raptor-youtube-playback-v9
+            - --power-test
+
+raptor-youtube-playback-h264-power-firefox:
+    description: "Raptor H264 YouTube Playback on Firefox with Power Testing"
+    try-name: raptor-youtube-playback-h264-power-firefox
+    treeherder-symbol: Rap(ytp-h264-p)
+    run-on-projects: ['mozilla-central']
+    max-run-time: 2700
+    tier: 3
+    mozharness:
+        extra-options:
+            - --test=raptor-youtube-playback-h264
+            - --power-test
+
 raptor-tp6-1-firefox-cold:
     description: "Raptor tp6-1 cold page-load on Firefox"
     try-name: raptor-tp6-1-firefox-cold
     treeherder-symbol: Rap(tp6-c-1)
     tier: 2
     mozharness:
         extra-options:
             - --test=raptor-tp6-cold-1
--- a/taskcluster/ci/test/test-platforms.yml
+++ b/taskcluster/ci/test/test-platforms.yml
@@ -297,16 +297,17 @@ macosx1014-64-shippable/opt:
     test-sets:
         - macosx1014-64-tests
         - macosx64-talos
         - desktop-screenshot-capture
         - awsy
         - raptor-chrome
         - raptor-chromium
         - raptor-firefox
+        - raptor-firefox-power
         - raptor-profiling
         - marionette-media-tests
         - web-platform-tests-wdspec-headless
 
 macosx1014-64/debug:
     build-platform: macosx64/debug
     test-sets:
         - macosx1014-64-tests
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -206,16 +206,22 @@ raptor-fetch-geckoview:
     - raptor-unity-webgl-refbrow
 
 raptor-fetch-chromium:
     - raptor-unity-webgl-chromium
     - raptor-wasm-misc-chromium
     - raptor-assorted-dom-chromium
     - raptor-jetstream2-chromium
 
+# Raptor desktop power tests
+# (requires Intel Power Gadget to be pre-installed)
+raptor-firefox-power:
+    - raptor-youtube-playback-v9-power-firefox
+    - raptor-youtube-playback-h264-power-firefox
+
 awsy:
     - awsy
     - awsy-base
     - awsy-dmd
     - awsy-base-dmd
     - awsy-tp6
 
 awsy-stylo-sequential:
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@@ -127,16 +127,17 @@ WINDOWS_WORKER_TYPES = {
       'virtual-with-gpu': 't-win10-64-gpu-s',
       'hardware': 't-win10-64-ref-hw',
     },
 }
 
 # os x worker types keyed by test-platform
 MACOSX_WORKER_TYPES = {
     'macosx1014-64': 'releng-hardware/gecko-t-osx-1014',
+    'macosx1014-64-power': 'releng-hardware/gecko-t-osx-1014-power'
 }
 
 
 def runs_on_central(test):
     return match_run_on_projects('mozilla-central', test['run-on-projects'])
 
 
 def fission_filter(test):
@@ -1380,17 +1381,20 @@ def set_worker_type(config, tests):
     for test in tests:
         # during the taskcluster migration, this is a bit tortured, but it
         # will get simpler eventually!
         test_platform = test['test-platform']
         if test.get('worker-type'):
             # This test already has its worker type defined, so just use that (yields below)
             pass
         elif test_platform.startswith('macosx1014-64'):
-            test['worker-type'] = MACOSX_WORKER_TYPES['macosx1014-64']
+            if '--power-test' in test['mozharness']['extra-options']:
+                test['worker-type'] = MACOSX_WORKER_TYPES['macosx1014-64-power']
+            else:
+                test['worker-type'] = MACOSX_WORKER_TYPES['macosx1014-64']
         elif test_platform.startswith('win'):
             # figure out what platform the job needs to run on
             if test['virtualization'] == 'hardware':
                 # some jobs like talos and reftest run on real h/w - those are all win10
                 if test_platform.startswith('windows10-64-ref-hw-2017'):
                     win_worker_type_platform = WINDOWS_WORKER_TYPES['windows10-64-ref-hw-2017']
                 elif test_platform.startswith('windows10-aarch64'):
                     win_worker_type_platform = WINDOWS_WORKER_TYPES['windows10-aarch64']
--- a/taskcluster/taskgraph/util/workertypes.py
+++ b/taskcluster/taskgraph/util/workertypes.py
@@ -22,16 +22,17 @@ WORKER_TYPES = {
     'scriptworker-k8s/gecko-3-beetmover': ('beetmover', None),
     'scriptworker-prov-v1/pushapk-v1': ('push-apk', None),
     "scriptworker-prov-v1/signing-linux-v1": ('scriptworker-signing', None),
     "scriptworker-k8s/gecko-3-shipit": ('shipit', None),
     "scriptworker-k8s/gecko-1-shipit": ('shipit', None),
     "scriptworker-prov-v1/treescript-v1": ('treescript', None),
     'terraform-packet/gecko-t-linux': ('docker-worker', 'linux'),
     'releng-hardware/gecko-t-osx-1014': ('generic-worker', 'macosx'),
+    'releng-hardware/gecko-t-osx-1014-power': ('generic-worker', 'macosx'),
 }
 
 
 @memoize
 def _get(graph_config, alias, level):
     """Get the configuration for this worker_type alias: {provisioner,
     worker-type, implementation, os}"""
     level = str(level)
--- a/testing/mozbase/mozpower/mozpower/intel_power_gadget.py
+++ b/testing/mozbase/mozpower/mozpower/intel_power_gadget.py
@@ -803,22 +803,22 @@ class IPGResultsHandler(object):
             average_utilization[utilized_name] = sum([
                 float(val)
                 for val in cut_results[utilization]
             ])/len(cut_results[utilization])
 
         summarized_results = {
             "utilization": {
                 "type": "power",
-                "test": str(test_name),
+                "test": str(test_name) + '-utilization',
                 "unit": "%",
                 "values": average_utilization
             },
             "power-usage": {
                 "type": "power",
-                "test": str(test_name),
+                "test": str(test_name) + '-cumulative',
                 "unit": "mWh",
                 "values": cumulative_mwh
             }
         }
 
         self._summarized_results = summarized_results
         return self._summarized_results
--- a/testing/mozbase/mozpower/tests/test_intelpowergadget.py
+++ b/testing/mozbase/mozpower/tests/test_intelpowergadget.py
@@ -220,17 +220,19 @@ def test_ipg_rh_format_to_perfherder_wit
     ipg_rh_obj.clean_ipg_data()
     formatted_data = ipg_rh_obj.format_ipg_data_to_partial_perfherder(
         1000, ipg_rh_obj._output_file_prefix
     )
 
     # Check that the expected entries exist
     assert len(formatted_data.keys()) == 2
     assert 'utilization' in formatted_data and 'power-usage' in formatted_data
-    assert formatted_data['utilization']['test'] == ipg_rh_obj._output_file_prefix
+    assert formatted_data['power-usage']['test'] == ipg_rh_obj._output_file_prefix + '-cumulative'
+    assert formatted_data['utilization']['test'] == \
+        ipg_rh_obj._output_file_prefix + '-utilization'
 
     # Check that gpu utilization doesn't exist but cpu does
     utilization_vals = formatted_data['utilization']['values']
     assert 'cpu' in utilization_vals
     assert 'gpu' not in utilization_vals
 
     expected_fields = ['processor-cores', 'processor-package', 'gpu', 'dram']
     consumption_vals = formatted_data['power-usage']['values']
--- a/testing/profiles/raptor/user.js
+++ b/testing/profiles/raptor/user.js
@@ -13,8 +13,11 @@ user_pref("dom.push.connection.enabled",
 
 // get the console logging out of the webext into the stdout
 user_pref("browser.dom.window.dump.enabled", true);
 user_pref("devtools.console.stdout.chrome", true);
 user_pref("devtools.console.stdout.content", true);
 
 // prevent pages from opening after a crash
 user_pref("browser.sessionstore.resume_from_crash", false);
+
+// prevent pages from opening after a crash
+user_pref("browser.sessionstore.resume_from_crash", false);
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@@ -91,16 +91,18 @@
 [include:tests/benchmarks/raptor-jetstream2.ini]
 [include:tests/benchmarks/raptor-motionmark-animometer.ini]
 [include:tests/benchmarks/raptor-motionmark-htmlsuite.ini]
 [include:tests/benchmarks/raptor-speedometer.ini]
 [include:tests/benchmarks/raptor-stylebench.ini]
 [include:tests/benchmarks/raptor-sunspider.ini]
 [include:tests/benchmarks/raptor-unity-webgl.ini]
 [include:tests/benchmarks/raptor-youtube-playback.ini]
+[include:tests/benchmarks/raptor-youtube-playback-v9.ini]
+[include:tests/benchmarks/raptor-youtube-playback-h264.ini]
 [include:tests/benchmarks/raptor-wasm-godot.ini]
 [include:tests/benchmarks/raptor-wasm-godot-baseline.ini]
 [include:tests/benchmarks/raptor-wasm-godot-ion.ini]
 [include:tests/benchmarks/raptor-wasm-godot-cranelift.ini]
 [include:tests/benchmarks/raptor-wasm-misc.ini]
 [include:tests/benchmarks/raptor-wasm-misc-baseline.ini]
 [include:tests/benchmarks/raptor-wasm-misc-ion.ini]
 [include:tests/benchmarks/raptor-wasm-misc-cranelift.ini]
--- a/testing/raptor/raptor/raptor.py
+++ b/testing/raptor/raptor/raptor.py
@@ -715,18 +715,19 @@ class RaptorDesktop(Raptor):
         # give our control server the browser process so it can shut it down later
         self.control_server.browser_proc = proc
 
     def run_test(self, test, timeout):
         # tests will be run warm (i.e. NO browser restart between page-cycles)
         # unless otheriwse specified in the test INI by using 'cold = true'
         mozpower_measurer = None
         if self.config.get('power_test', False):
-            output_dir = os.path.join(self.artifact_dir, 'power-measurements')
-            test_dir = os.path.join(output_dir, test['name'].replace('/', '-').replace('\\', '-'))
+            powertest_name = test['name'].replace('/', '-').replace('\\', '-')
+            output_dir = os.path.join(self.artifact_dir, 'power-measurements-%s' % powertest_name)
+            test_dir = os.path.join(output_dir, powertest_name)
 
             try:
                 if not os.path.exists(output_dir):
                     os.mkdir(output_dir)
                 if not os.path.exists(test_dir):
                     os.mkdir(test_dir)
             except Exception:
                 LOG.critical("Could not create directories to store power testing data.")
@@ -749,18 +750,22 @@ class RaptorDesktop(Raptor):
 
         if mozpower_measurer:
             mozpower_measurer.finalize_power_measurements(test_name=test['name'])
             perfherder_data = mozpower_measurer.get_perfherder_data()
 
             if not self.config.get('run_local', False):
                 # when not running locally, zip the data and delete the folder which
                 # was placed in the zip
-                power_data_path = os.path.join(self.artifact_dir, 'power-measurements')
-                shutil.make_archive(power_data_path + '.zip', 'zip', power_data_path)
+                powertest_name = test['name'].replace('/', '-').replace('\\', '-')
+                power_data_path = os.path.join(
+                    self.artifact_dir,
+                    'power-measurements-%s' % powertest_name
+                )
+                shutil.make_archive(power_data_path, 'zip', power_data_path)
                 shutil.rmtree(power_data_path)
 
             self.control_server.submit_supporting_data(perfherder_data['utilization'])
             self.control_server.submit_supporting_data(perfherder_data['power-usage'])
 
     def __run_test_cold(self, test, timeout):
         '''
         Run the Raptor test but restart the entire browser app between page-cycles.
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/benchmarks/raptor-youtube-playback-h264.ini
@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Fullscreen YouTube-playback performance benchmark
+#
+# Original location of source and media files:
+#   https://ytlr-cert.appspot.com/2019/main.html?test_type=playbackperf-test
+
+[DEFAULT]
+apps = firefox
+type = benchmark
+use_live_sites = true
+page_cycles = 20
+# account for a page-cycle-duration of at maximum 45 minutes
+page_timeout = 2700000
+alert_threshold = 2.0
+lower_is_better = true
+unit = score
+subtest_lower_is_better = true
+subtest_unit = score
+
+[raptor-youtube-playback-h264-1080p30-firefox]
+test_url = http://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-test&raptor=true&command=run&tests=16&muted=true
+
+[raptor-youtube-playback-h264-1080p60-firefox]
+test_url = http://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-test&raptor=true&command=run&tests=24&muted=true
+
+[raptor-youtube-playback-h264-full-1080p30-firefox]
+test_url = http://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-test&raptor=true&command=run&tests=16&muted=true&fullscreen=true
+preferences = {"full-screen-api.allow-trusted-requests-only": false,
+               "full-screen-api.warning.timeout": 0}
+
+[raptor-youtube-playback-h264-full-1080p60-firefox]
+test_url = http://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-test&raptor=true&command=run&tests=24&muted=true&fullscreen=true
+preferences = {"full-screen-api.allow-trusted-requests-only": false,
+               "full-screen-api.warning.timeout": 0}
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/benchmarks/raptor-youtube-playback-v9.ini
@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Fullscreen YouTube-playback performance benchmark
+#
+# Original location of source and media files:
+#   https://ytlr-cert.appspot.com/2019/main.html?test_type=playbackperf-test
+
+[DEFAULT]
+apps = firefox
+type = benchmark
+use_live_sites = true
+page_cycles = 20
+# account for a page-cycle-duration of at maximum 45 minutes
+page_timeout = 2700000
+alert_threshold = 2.0
+lower_is_better = true
+unit = score
+subtest_lower_is_better = true
+subtest_unit = score
+
+[raptor-youtube-playback-v9-1080p30-firefox]
+test_url = http://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-test&raptor=true&command=run&tests=8&muted=true
+
+[raptor-youtube-playback-v9-1080p60-firefox]
+test_url = http://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-test&raptor=true&command=run&tests=20&muted=true
+
+[raptor-youtube-playback-v9-full-1080p30-firefox]
+test_url = http://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-test&raptor=true&command=run&tests=8&muted=true&fullscreen=true
+preferences = {"full-screen-api.allow-trusted-requests-only": false,
+               "full-screen-api.warning.timeout": 0}
+
+[raptor-youtube-playback-v9-full-1080p60-firefox]
+test_url = http://yttest.prod.mozaws.net/2019/main.html?test_type=playbackperf-test&raptor=true&command=run&tests=20&muted=true&fullscreen=true
+preferences = {"full-screen-api.allow-trusted-requests-only": false,
+               "full-screen-api.warning.timeout": 0}