Bug 1541189 - Fix intermittents on stream test - r=whimboo
authorTarek Ziadé <tarek@mozilla.com>
Mon, 08 Apr 2019 13:04:20 +0000
changeset 468344 db1c74f43f924149f4790858c83d1803e0534846
parent 468343 b88166b3b314cda6ea94a0444d116c6b0321b6bd
child 468345 8de77b46cd56f80c3d19fc3cbf7eff2af0cc2817
push id35833
push userdvarga@mozilla.com
push dateMon, 08 Apr 2019 16:16:26 +0000
treeherdermozilla-central@50ce9167f1ce [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerswhimboo
bugs1541189
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1541189 - Fix intermittents on stream test - r=whimboo Tweak the Streaming test to fix intermittents. Differential Revision: https://phabricator.services.mozilla.com/D26288
.eslintignore
dom/media/test/marionette/test_youtube.py
dom/media/test/marionette/yttest/debug_info.js
dom/media/test/marionette/yttest/duration_test.js
dom/media/test/marionette/yttest/playback.py
dom/media/test/marionette/yttest/support.py
dom/media/test/marionette/yttest/uR0N3DrybGQ.manifest
dom/media/test/marionette/yttest/until_end_test.js
dom/media/test/marionette/yttest/video_playback_quality.js
dom/media/test/marionette/yttest/ytpage.py
--- a/.eslintignore
+++ b/.eslintignore
@@ -166,16 +166,17 @@ dom/encoding/**
 dom/events/**
 dom/fetch/**
 dom/file/**
 dom/flex/**
 dom/grid/**
 dom/html/**
 dom/jsurl/**
 dom/media/test/**
+!dom/media/test/marionette/yttest/*.js
 dom/media/tests/**
 dom/media/webaudio/**
 dom/media/webspeech/**
 dom/messagechannel/**
 dom/midi/**
 dom/network/**
 dom/payments/**
 dom/performance/**
--- a/dom/media/test/marionette/test_youtube.py
+++ b/dom/media/test/marionette/test_youtube.py
@@ -6,23 +6,17 @@ import os
 
 sys.path.append(os.path.dirname(__file__))
 from yttest.support import VideoStreamTestCase
 
 
 class YoutubeTest(VideoStreamTestCase):
 
     # bug 1513511
-    def test_stream_30_seconds(self):
-        # XXX use the VP9 video we will settle on.
-        with self.youtube_video("BZP1rYjoBgI") as page:
+    def test_stream_4K(self):
+        with self.youtube_video("uR0N3DrybGQ", duration=15) as page:
             res = page.run_test()
-            self.assertTrue(res is not None, "We did not get back the results")
-            self.assertLess(res["droppedVideoFrames"], res["totalVideoFrames"] * 0.04)
-            # extracting in/out from the debugInfo
-            video_state = res["debugInfo"][7]
-            video_in = int(video_state.split(" ")[10].split("=")[-1])
-            video_out = int(video_state.split(" ")[11].split("=")[-1])
-            # what's the ratio ? we want 99%+
-            if video_out == video_in:
-                return
-            in_out_ratio = float(video_out) / float(video_in) * 100
-            self.assertMore(in_out_ratio, 99.0)
+            self.assertVideoQuality(res)
+
+    def test_stream_480p(self):
+        with self.youtube_video("BZP1rYjoBgI", duration=15) as page:
+            res = page.run_test()
+            self.assertVideoQuality(res)
--- a/dom/media/test/marionette/yttest/debug_info.js
+++ b/dom/media/test/marionette/yttest/debug_info.js
@@ -1,18 +1,18 @@
 video.mozRequestDebugInfo().then(debugInfo => {
+  // The parsing won't be necessary once we have bug 1542674
   try {
     debugInfo = debugInfo.replace(/\t/g, '').split(/\n/g);
     var JSONDebugInfo = "{";
       for(let g =0; g<debugInfo.length-1; g++){
           var pair = debugInfo[g].split(": ");
           JSONDebugInfo += '"' + pair[0] + '":"' + pair[1] + '",';
       }
       JSONDebugInfo = JSONDebugInfo.slice(0,JSONDebugInfo.length-1);
       JSONDebugInfo += "}";
-      result["debugInfo"] = JSON.parse(JSONDebugInfo);
+      result["mozRequestDebugInfo"] = JSON.parse(JSONDebugInfo);
   } catch (err) {
     console.log(`Error '${err.toString()} in JSON.parse(${debugInfo})`);
-    result["debugInfo"] = debugInfo;
+    result["mozRequestDebugInfo"] = debugInfo;
   }
-  result["debugInfo"] = debugInfo;
   resolve(result);
 });
--- a/dom/media/test/marionette/yttest/duration_test.js
+++ b/dom/media/test/marionette/yttest/duration_test.js
@@ -5,17 +5,19 @@ const resolve = arguments[arguments.leng
 // this script is injected by marionette to collect metrics
 var video = document.getElementsByTagName("video")[0];
 if (!video) {
   return "Can't find the video tag";
 }
 
 video.addEventListener("timeupdate", () => {
     if (video.currentTime >= %(duration)s) {
-      video.pause();
       %(video_playback_quality)s
       %(debug_info)s
+      // Pausing after we get the debug info so
+      // we can also look at in/out data in buffers
+      video.pause();
     }
   }
 );
 
 video.play();
 
--- a/dom/media/test/marionette/yttest/playback.py
+++ b/dom/media/test/marionette/yttest/playback.py
@@ -7,16 +7,21 @@ MITM Script used to play back media file
 This is a self-contained script that should not import anything else
 except modules from the standard library and mitmproxy modules.
 """
 import os
 import sys
 import datetime
 import time
 
+try:
+    from urllib import unquote
+except ImportError:
+    from urllib.parse import unquote
+
 
 itags = {
     "5": {
         "Extension": "flv",
         "Resolution": "240p",
         "VideoEncoding": "Sorenson H.283",
         "AudioEncoding": "mp3",
         "Itag": 5,
@@ -591,31 +596,41 @@ def OK(flow, code=204):
     """
     from mitmproxy import http
 
     flow.error = None
     flow.response = http.HTTPResponse(b"HTTP/1.1", code, b"OK", {}, b"")
 
 
 def request(flow):
+    # in some cases, the YT client sends requests with a methode of the form:
+    #   VAR=XX%3GET /xxx
+    # this will clean it up:
+    method = flow.request.method
+    method = unquote(method).split("=")
+    flow.request.method = method[-1]
+
     # All requests made for stats purposes can be discarded and
     # a 204 sent back to the client.
     if flow.request.url.startswith("https://www.youtube.com/ptracking"):
         OK(flow)
         return
     if flow.request.url.startswith("https://www.youtube.com/api/stats/playback"):
         OK(flow)
         return
     if flow.request.url.startswith("https://www.youtube.com/api/stats/watchtime"):
         OK(flow)
         return
     # disable a few trackers, sniffers, etc
     if "push.services.mozilla.com" in flow.request.url:
         OK(flow, code=200)
         return
+    if "tracking-protection.cdn.mozilla.net" in flow.request.url:
+        OK(flow, code=200)
+        return
     if "gen_204" in flow.request.url:
         OK(flow)
         return
 
     # we don't want to post back any data, discarding.
     if flow.request.method == "POST":
         OK(flow)
         return
--- a/dom/media/test/marionette/yttest/support.py
+++ b/dom/media/test/marionette/yttest/support.py
@@ -18,16 +18,17 @@ playback_script = os.path.join(here, "pl
 
 
 class VideoStreamTestCase(MarionetteTestCase):
     def setUp(self):
         MarionetteTestCase.setUp(self)
         if "MOZ_UPLOAD_DIR" not in os.environ:
             os.environ["OBJ_PATH"] = "/tmp/"
         self.marionette.set_pref("media.autoplay.default", 1)
+        self.marionette.set_pref("privacy.trackingprotection.enabled", False)
 
     @contextmanager
     def using_proxy(self, video_id):
         config = {}
         config["binary"] = self.marionette.bin
         config["app"] = "firefox"
         config["platform"] = mozinfo.os
         config["processor"] = mozinfo.processor
@@ -51,18 +52,16 @@ class VideoStreamTestCase(MarionetteTest
         # so we don't have to ask amozproxy tool user to provide this:
         config[
             "playback_binary_manifest"
         ] = "mitmproxy-rel-bin-4.0.4-{platform}.manifest"
 
         playback_file = os.path.join(playback_dir, "%s.playback" % video_id)
 
         config["playback_tool_args"] = [
-            "--set",
-            "stream_large_bodies=30",
             "--ssl-insecure",
             "--server-replay-nopop",
             "--set",
             "upstream_cert=false",
             "-S",
             playback_file,
             "-s",
             playback_script,
@@ -77,14 +76,34 @@ class VideoStreamTestCase(MarionetteTest
             yield proxy
         finally:
             proxy.stop()
 
     @contextmanager
     def youtube_video(self, video_id, **options):
         proxy = options.get("proxy", True)
         if proxy:
-            with self.using_proxy(video_id):
+            with self.using_proxy(video_id) as proxy:
+                options["upload_dir"] = proxy.upload_dir
                 with using_page(video_id, self.marionette, **options) as page:
                     yield page
         else:
             with using_page(video_id, self.marionette, **options) as page:
                 yield page
+
+    def assertVideoQuality(self, res):
+        self.assertTrue(res is not None, "We did not get back the results")
+        debug_info = res["mozRequestDebugInfo"]
+
+        # looking at mNumSamplesOutputTotal vs mNumSamplesSkippedTotal
+        decoded, skipped = debug_info["Video Frames Decoded"].split(" ", 1)
+        decoded = int(decoded)
+        skipped = int(skipped.split("=")[-1][:-1])
+        self.assertLess(skipped, decoded * 0.04)
+
+        # extracting in/out from the debugInfo
+        video_state = debug_info["Video State"]
+        video_in = int(video_state["in"])
+        video_out = int(video_state["out"])
+        # what's the ratio ? we want 99%+
+        if video_out != video_in:
+            in_out_ratio = float(video_out) / float(video_in) * 100
+            self.assertGreater(in_out_ratio, 99.0)
new file mode 100644
--- /dev/null
+++ b/dom/media/test/marionette/yttest/uR0N3DrybGQ.manifest
@@ -0,0 +1,10 @@
+[
+  {
+    "size": 629013569,
+    "visibility": "public",
+    "digest": "213afa0e40411c26c86092a0803099a8c596b27cf789ed658ba0cf50dd8b404926dd784cd0236922aca22d3763edff666dd247c14bfe38359fb9d767f1869048",
+    "algorithm": "sha512",
+    "filename": "uR0N3DrybGQ.tar.gz",
+    "unpack": true
+  }
+]
--- a/dom/media/test/marionette/yttest/until_end_test.js
+++ b/dom/media/test/marionette/yttest/until_end_test.js
@@ -4,15 +4,17 @@ const resolve = arguments[arguments.leng
 
 // this script is injected by marionette to collect metrics
 var video = document.getElementsByTagName("video")[0];
 if (!video) {
   return "Can't find the video tag";
 }
 
 video.addEventListener("ended", () => {
-    video.pause();
     %(video_playback_quality)s
     %(debug_info)s
+    // Pausing after we get the debug info so
+    // we can also look at in/out data in buffers
+    video.pause();
   }, {once: true}
 );
 
 video.play();
--- a/dom/media/test/marionette/yttest/video_playback_quality.js
+++ b/dom/media/test/marionette/yttest/video_playback_quality.js
@@ -1,7 +1,1 @@
-var vpq = video.getVideoPlaybackQuality();
-var result = {"currentTime": video.currentTime};
-result["creationTime"] = vpq.creationTime;
-result["corruptedVideoFrames"] = vpq.corruptedVideoFrames;
-result["droppedVideoFrames"] = vpq.droppedVideoFrames;
-result["totalVideoFrames"] = vpq.totalVideoFrames;
-result["defaultPlaybackRate"] = video.playbackRate;
+var result = {"getVideoPlaybackQuality": video.getVideoPlaybackQuality()};
--- a/dom/media/test/marionette/yttest/ytpage.py
+++ b/dom/media/test/marionette/yttest/ytpage.py
@@ -1,16 +1,21 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 """
 Drives the browser during the playback test.
 """
 import contextlib
 import os
+import time
+import json
+import re
+
+from marionette_driver.by import By
 
 
 here = os.path.dirname(__file__)
 js = os.path.join(here, "until_end_test.js")
 with open(js) as f:
     UNTIL_END_TEST = f.read()
 
 js = os.path.join(here, "duration_test.js")
@@ -18,16 +23,29 @@ with open(js) as f:
     DURATION_TEST = f.read()
 
 JS_MACROS = {"video_playback_quality": "", "debug_info": "", "force_hd": ""}
 for script in JS_MACROS:
     js = os.path.join(here, "%s.js" % script)
     with open(js) as f:
         JS_MACROS[script] = f.read()
 
+SPLIT_FIELD = (
+    "Audio State",
+    "Audio Track Buffer Details",
+    "AudioSink",
+    "MDSM",
+    "Video State",
+    "Video Track Buffer Details",
+    "Dumping Audio Track",
+    "Dumping Video Track",
+    "MediaDecoder",
+    "VideoSink",
+)
+
 
 class YoutubePage:
     def __init__(self, video_id, marionette, **options):
         self.video_id = video_id
         self.marionette = marionette
         self.url = "https://www.youtube.com/watch?v=%s" % self.video_id
         self.started = False
         self.capabilities = {
@@ -48,31 +66,81 @@ class YoutubePage:
     def start_video(self):
         self.marionette.start_session(self.capabilities)
         self.marionette.timeout.script = 600
         self.marionette.navigate(self.url)
         self.started = True
 
     def run_test(self):
         self.start_video()
+        # If we don't pause here for just a bit the media events
+        # are not intercepted.
+        time.sleep(5)
+        body = self.marionette.find_element(By.TAG_NAME, "html")
+        body.click()
         options = dict(JS_MACROS)
         options.update(self.options)
         if "duration" in options:
             script = DURATION_TEST % options
         else:
             script = UNTIL_END_TEST % options
-        self.marionette.set_pref("media.autoplay.default", 0)
-        return self.execute_async_script(script)
+        res = self.execute_async_script(script)
+        if res is None:
+            return res
+        res = self._parse_res(res)
+        self._dump_res(res)
+        return res
 
     def execute_async_script(self, script, context=None):
         if context is None:
             context = self.marionette.CONTEXT_CONTENT
         with self.marionette.using_context(context):
             return self.marionette.execute_async_script(script, sandbox="system")
 
+    def _parse_res(self, res):
+        debug_info = {}
+        # The parsing won't be necessary once we have bug 1542674
+        for key, value in res["mozRequestDebugInfo"].items():
+            key, value = key.strip(), value.strip()
+            if key.startswith(SPLIT_FIELD):
+                value_dict = {}
+                for field in re.findall(r"\S+\(.+\)\s|\S+", value):
+                    field = field.strip()
+                    if field == "":
+                        continue
+                    if field.startswith("VideoQueue"):
+                        k = "VideoQueue"
+                        v = field[len("VideoQueue(") : -2]  # noqa: E203
+                        fields = {}
+                        v = v.split(" ")
+                        for h in v:
+                            f, vv = h.split("=")
+                            fields[f] = vv
+                        v = fields
+                    else:
+                        if "=" in field:
+                            k, v = field.split("=", 1)
+                        else:
+                            k, v = field.split(":", 1)
+                    value_dict[k] = v
+                value = value_dict
+            debug_info[key] = value
+        res["mozRequestDebugInfo"] = debug_info
+        return res
+
+    def _dump_res(self, res):
+        raw = json.dumps(res, indent=2, sort_keys=True)
+        print(raw)
+        if "upload_dir" in self.options:
+            fn = "%s-videoPlaybackQuality.json" % self.video_id
+            fn = os.path.join(self.options["upload_dir"], fn)
+            # dumping on disk
+            with open(fn, "w") as f:
+                f.write(raw)
+
     def close(self):
         if self.started:
             self.marionette.delete_session()
 
 
 @contextlib.contextmanager
 def using_page(video_id, marionette, **options):
     page = YoutubePage(video_id, marionette, **options)