Bug 1548845 - [raptor] Integrate Youtube video playback performance suite as benchmark test. r=perftest-reviewers,rwood
authorHenrik Skupin <mail@hskupin.info>
Fri, 17 May 2019 20:16:12 +0000
changeset 474450 a757325b769035a6b9d395e524d80070b92432cf
parent 474449 4a74609752d2e77e4be401e92978c9c32a842b40
child 474451 0a796fa7c16ff7147cd5c570b2cae5afa670edcc
push id36037
push usermalexandru@mozilla.com
push dateMon, 20 May 2019 09:51:50 +0000
treeherdermozilla-central@17d1c1e26e31 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersperftest-reviewers, rwood
bugs1548845
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1548845 - [raptor] Integrate Youtube video playback performance suite as benchmark test. r=perftest-reviewers,rwood Differential Revision: https://phabricator.services.mozilla.com/D30483
testing/raptor/raptor/output.py
testing/raptor/raptor/raptor.ini
testing/raptor/raptor/tests/raptor-youtube-playback.ini
testing/raptor/webext/raptor/manifest.json
--- a/testing/raptor/raptor/output.py
+++ b/testing/raptor/raptor/output.py
@@ -126,32 +126,34 @@ class Output(object):
                             new_subtest['shouldAlert'] = True
 
                     new_subtest['value'] = filters.median(filtered_values)
 
                     vals.append([new_subtest['value'], new_subtest['name']])
                     subtests.append(new_subtest)
 
             elif test.type == "benchmark":
-                if 'speedometer' in test.measurements:
-                    subtests, vals = self.parseSpeedometerOutput(test)
+                if 'assorted-dom' in test.measurements:
+                    subtests, vals = self.parseAssortedDomOutput(test)
                 elif 'motionmark' in test.measurements:
                     subtests, vals = self.parseMotionmarkOutput(test)
+                elif 'speedometer' in test.measurements:
+                    subtests, vals = self.parseSpeedometerOutput(test)
                 elif 'sunspider' in test.measurements:
                     subtests, vals = self.parseSunspiderOutput(test)
+                elif 'unity-webgl' in test.measurements:
+                    subtests, vals = self.parseUnityWebGLOutput(test)
+                elif 'wasm-godot' in test.measurements:
+                    subtests, vals = self.parseWASMGodotOutput(test)
+                elif 'wasm-misc' in test.measurements:
+                    subtests, vals = self.parseWASMMiscOutput(test)
                 elif 'webaudio' in test.measurements:
                     subtests, vals = self.parseWebaudioOutput(test)
-                elif 'unity-webgl' in test.measurements:
-                    subtests, vals = self.parseUnityWebGLOutput(test)
-                elif 'assorted-dom' in test.measurements:
-                    subtests, vals = self.parseAssortedDomOutput(test)
-                elif 'wasm-misc' in test.measurements:
-                    subtests, vals = self.parseWASMMiscOutput(test)
-                elif 'wasm-godot' in test.measurements:
-                    subtests, vals = self.parseWASMGodotOutput(test)
+                elif 'youtube-playbackperf-test' in test.measurements:
+                    subtests, vals = self.parseYoutubePlaybackPerformanceOutput(test)
                 suite['subtests'] = subtests
 
             else:
                 LOG.error("output.summarize received unsupported test results type for %s" %
                           test.name)
                 return
 
             # for benchmarks there is generally  more than one subtest in each cycle
@@ -696,16 +698,82 @@ class Output(object):
             _subtests[name]['value'] = round(filters.median(_subtests[name]['replicates']), 2)
             subtests.append(_subtests[name])
             # only use the 'total's to compute the overall result
             if name == 'total':
                 vals.append([_subtests[name]['value'], name])
 
         return subtests, vals
 
+    def parseYoutubePlaybackPerformanceOutput(self, test):
+        """Parse the metrics for the Youtube playback performance test.
+
+        For each video measured values for dropped and decoded frames will be
+        available from the benchmark site.
+
+        {u'PlaybackPerf.VP9.2160p60@2X': {u'droppedFrames': 1, u'decodedFrames': 796}
+
+        With each page cycle / iteration of the test multiple values can be present.
+
+        Raptor will calculate the percentage of dropped frames to decoded frames.
+        All those three values will then be emitted as separate sub tests.
+        """
+        _subtests = {}
+        data = test.measurements['youtube-playbackperf-test']
+
+        def create_subtest_entry(name, value,
+                                 unit=test.subtest_unit,
+                                 lower_is_better=test.subtest_lower_is_better):
+            # build a list of subtests and append all related replicates
+            if name not in _subtests.keys():
+                # subtest not added yet, first pagecycle, so add new one
+                _subtests[name] = {
+                    'name': name,
+                    'unit': unit,
+                    'lowerIsBetter': lower_is_better,
+                    'replicates': [],
+                }
+
+            _subtests[name]['replicates'].append(value)
+
+        for pagecycle in data:
+            for _sub, _value in pagecycle[0].iteritems():
+                try:
+                    percent_dropped = float(_value['droppedFrames']) / _value['decodedFrames']
+                except ZeroDivisionError:
+                    # if no frames have been decoded the playback failed completely
+                    percent_dropped = 1
+
+                # Remove the not needed "PlaybackPerf." prefix from each test
+                _sub = _sub.split('PlaybackPerf.', 1)[-1]
+
+                # build a list of subtests and append all related replicates
+                create_subtest_entry("{}_decoded_frames".format(_sub),
+                                     _value['decodedFrames'],
+                                     lower_is_better=False,
+                                     )
+                create_subtest_entry("{}_dropped_frames".format(_sub),
+                                     _value['droppedFrames'],
+                                     )
+                create_subtest_entry("{}_%_dropped_frames".format(_sub),
+                                     percent_dropped,
+                                     )
+
+        vals = []
+        subtests = []
+        names = _subtests.keys()
+        names.sort(reverse=True)
+        for name in names:
+            _subtests[name]['value'] = round(filters.median(_subtests[name]['replicates']), 2)
+            subtests.append(_subtests[name])
+            if name.endswith("dropped_frames"):
+                vals.append([_subtests[name]['value'], name])
+
+        return subtests, vals
+
     def summarize_screenshots(self, screenshots):
         if len(screenshots) == 0:
             return
 
         self.summarized_screenshots.append("""<!DOCTYPE html>
         <head>
         <style>
             table, th, td {
@@ -949,16 +1017,22 @@ class Output(object):
         return sum(results)
 
     @classmethod
     def assorted_dom_score(cls, val_list):
         results = [i for i, j in val_list]
         return round(filters.geometric_mean(results), 2)
 
     @classmethod
+    def youtube_playback_performance_score(cls, val_list):
+        """Calculate percentage of failed tests."""
+        results = [i for i, j in val_list]
+        return round(filters.mean(results), 2)
+
+    @classmethod
     def supporting_data_total(cls, val_list):
         results = [i for i, j in val_list]
         return sum(results)
 
     def construct_summary(self, vals, testname):
         if testname.startswith('raptor-v8_7'):
             return self.v8_Metric(vals)
         elif testname.startswith('raptor-kraken'):
@@ -976,14 +1050,16 @@ class Output(object):
         elif testname.startswith('raptor-webaudio'):
             return self.webaudio_score(vals)
         elif testname.startswith('raptor-assorted-dom'):
             return self.assorted_dom_score(vals)
         elif testname.startswith('raptor-wasm-misc'):
             return self.wasm_misc_score(vals)
         elif testname.startswith('raptor-wasm-godot'):
             return self.wasm_godot_score(vals)
+        elif testname.startswith('raptor-youtube-playback'):
+            return self.youtube_playback_performance_score(vals)
         elif testname.startswith('supporting_data'):
             return self.supporting_data_total(vals)
         elif len(vals) > 1:
             return round(filters.geometric_mean([i for i, j in vals]), 2)
         else:
             return round(filters.mean([i for i, j in vals]), 2)
--- a/testing/raptor/raptor/raptor.ini
+++ b/testing/raptor/raptor/raptor.ini
@@ -50,16 +50,17 @@
 # raptor benchmark tests
 [include:tests/raptor-assorted-dom.ini]
 [include:tests/raptor-motionmark-animometer.ini]
 [include:tests/raptor-motionmark-htmlsuite.ini]
 [include:tests/raptor-speedometer.ini]
 [include:tests/raptor-stylebench.ini]
 [include:tests/raptor-sunspider.ini]
 [include:tests/raptor-unity-webgl.ini]
+[include:tests/raptor-youtube-playback.ini]
 [include:tests/raptor-wasm-godot.ini]
 [include:tests/raptor-wasm-godot-baseline.ini]
 [include:tests/raptor-wasm-godot-ion.ini]
 [include:tests/raptor-wasm-godot-cranelift.ini]
 [include:tests/raptor-wasm-misc.ini]
 [include:tests/raptor-wasm-misc-baseline.ini]
 [include:tests/raptor-wasm-misc-ion.ini]
 [include:tests/raptor-wasm-misc-cranelift.ini]
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/tests/raptor-youtube-playback.ini
@@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Youtube playback performance benchmark
+#
+# Original location of source and media files:
+#   https://ytlr-cert.appspot.com/2019/main.html?test_type=playbackperf-test
+
+[DEFAULT]
+type =  benchmark
+use_live_sites = true
+gecko_profile_interval = 1
+gecko_profile_entries = 14000000
+gecko_profile_threads = MediaPlayback
+test_url = http://yttest.dev.mozaws.net/2019/main.html?test_type=playbackperf-test&raptor=true&command=run&exclude=1,2
+page_cycles = 1
+# account for a page cycle duration of at maximum 45 minutes
+page_timeout = 2700000
+alert_threshold = 2.0
+lower_is_better = true
+unit = score
+subtest_lower_is_better = true
+subtest_unit = score
+# TODO: Allow the host / port option in the manifest (Bug 1547932)
+preferences = {"network.proxy.type": 0}
+
+[raptor-youtube-playback-firefox]
+apps = firefox
--- a/testing/raptor/webext/raptor/manifest.json
+++ b/testing/raptor/webext/raptor/manifest.json
@@ -47,25 +47,28 @@
                   "*://*.yahoo.com/*",
                   "*://*.youtube.com/*",
                   "*://*.yandex.ru/*"
                   ],
       "js": ["pageload.js"],
       "run_at": "document_end"
     },
     {
-      "matches": ["*://*/Speedometer/index.html*",
-                  "*://*/StyleBench/*",
-                  "*://*/MotionMark/*",
-                  "*://*/SunSpider/*",
-                  "*://*/webaudio/*",
-                  "*://*/unity-webgl/index.html*",
-                  "*://*/wasm-misc/index.html*",
-                  "*://*/wasm-godot/index.html*",
-                  "*://*/assorted-dom/assorted/results.html*"],
+      "matches": [
+        "*://*/Speedometer/index.html*",
+        "*://*/StyleBench/*",
+        "*://*/MotionMark/*",
+        "*://*/SunSpider/*",
+        "*://*/webaudio/*",
+        "*://*/unity-webgl/index.html*",
+        "*://*/wasm-misc/index.html*",
+        "*://*/wasm-godot/index.html*",
+        "*://*/assorted-dom/assorted/results.html*",
+        "*://*.mozaws.net/*"
+      ],
       "js": ["benchmark.js"],
       "run_at": "document_end"
     }
   ],
   "browser_action": {
     "browser_style": true,
     "default_icon": "icon.png",
     "default_title": "Raptor LOADED"