Bug 610570 - Only skip to next keyframe when not running out of data to decode. r=roc a=blocking2.0
authorChris Pearce <chris@pearce.org.nz>
Mon, 29 Nov 2010 09:06:38 +1300
changeset 58312 7db5a5d297c67361a110e1b947d017a17e00f7a6
parent 58311 f5cf523d8bd23a60b684276a94cebfd2ae96a235
child 58313 00ea82fd6c59e927f4d78d4d5a78fd2a3329ff31
push id1
push usershaver@mozilla.com
push dateTue, 04 Jan 2011 17:58:04 +0000
reviewersroc, blocking2
bugs610570
milestone2.0b8pre
Bug 610570 - Only skip to next keyframe when not running out of data to decode. r=roc a=blocking2.0
content/media/nsBuiltinDecoderStateMachine.cpp
content/media/nsBuiltinDecoderStateMachine.h
--- a/content/media/nsBuiltinDecoderStateMachine.cpp
+++ b/content/media/nsBuiltinDecoderStateMachine.cpp
@@ -64,53 +64,64 @@ extern PRLogModuleInfo* gBuiltinDecoderL
 // on the download rate. BUFFERING_MIN_RATE is the minimum download
 // rate to be used in that calculation to help avoid constant buffering
 // attempts at a time when the average download rate has not stabilised.
 #define BUFFERING_MIN_RATE 50000
 #define BUFFERING_RATE(x) ((x)< BUFFERING_MIN_RATE ? BUFFERING_MIN_RATE : (x))
 
 // If audio queue has less than this many ms of decoded audio, we won't risk
 // trying to decode the video, we'll skip decoding video up to the next
-// keyframe.
-//
-// Also if the decode catches up with the end of the downloaded data,
-// we'll only go into BUFFERING state if we've got audio and have queued
-// less than LOW_AUDIO_MS of audio, or if we've got video and have queued
-// less than LOW_VIDEO_FRAMES frames.
+// keyframe. We may increase this value for an individual decoder if we
+// encounter video frames which take a long time to decode.
 static const PRUint32 LOW_AUDIO_MS = 300;
 
 // If more than this many ms of decoded audio is queued, we'll hold off
-// decoding more audio.
-const unsigned AMPLE_AUDIO_MS = 2000;
+// decoding more audio. If we increase the low audio threshold (see
+// LOW_AUDIO_MS above) we'll also increase this value to ensure it's not
+// less than the low audio threshold.
+const unsigned AMPLE_AUDIO_MS = 1000;
 
 // Maximum number of bytes we'll allocate and write at once to the audio
 // hardware when the audio stream contains missing samples and we're
 // writing silence in order to fill the gap. We limit our silence-writes
 // to 32KB in order to avoid allocating an impossibly large chunk of
 // memory if we encounter a large chunk of silence.
 const PRUint32 SILENCE_BYTES_CHUNK = 32 * 1024;
 
 // If we have fewer than LOW_VIDEO_FRAMES decoded frames, and
 // we're not "pumping video", we'll skip the video up to the next keyframe
 // which is at or after the current playback position.
-//
-// Also if the decode catches up with the end of the downloaded data,
-// we'll only go into BUFFERING state if we've got audio and have queued
-// less than LOW_AUDIO_MS of audio, or if we've got video and have queued
-// less than LOW_VIDEO_FRAMES frames.
 static const PRUint32 LOW_VIDEO_FRAMES = 1;
 
 // If we've got more than AMPLE_VIDEO_FRAMES decoded video frames waiting in
 // the video queue, we will not decode any more video frames until some have
 // been consumed by the play state machine thread.
 static const PRUint32 AMPLE_VIDEO_FRAMES = 10;
 
 // Arbitrary "frame duration" when playing only audio.
 static const int AUDIO_DURATION_MS = 40;
 
+// If we increase our "low audio threshold" (see LOW_AUDIO_MS above), we
+// use this as a factor in all our calculations. Increasing this will cause
+// us to be more likely to increase our low audio threshold, and to
+// increase it by more.
+static const int THRESHOLD_FACTOR = 2;
+
+// Number of milliseconds worth of estimated data we'll try to maintain
+// ahead of the decoder position when playing non-live streams. If the
+// decoder position catches up with the download and comes within this
+// many ms of estimated data, we'll stop playback and start to buffer.
+static const double NORMAL_BUFFER_MARGIN = 100.0;
+
+// Arbitrary number of bytes we try to keep buffered ahead of the download
+// position when playing a live or non-seekable stream. When playing a live
+// or non-seekable stream, if we have less than this amount of downloaded
+// undecoded data, we'll stop playback and start buffering.
+static const int LIVE_BUFFER_MARGIN = 100000;
+
 class nsAudioMetadataEventRunner : public nsRunnable
 {
 private:
   nsCOMPtr<nsBuiltinDecoder> mDecoder;
 public:
   nsAudioMetadataEventRunner(nsBuiltinDecoder* aDecoder, PRUint32 aChannels,
                              PRUint32 aRate, PRUint32 aFrameBufferLength) :
     mDecoder(aDecoder),
@@ -207,16 +218,25 @@ void nsBuiltinDecoderStateMachine::Decod
   // no longer be considered to be "pumping video".
   const unsigned videoPumpThreshold = AMPLE_VIDEO_FRAMES / 2;
 
   // After the audio decode fills with more than audioPumpThresholdMs ms
   // of decoded audio, we'll start to check whether the audio or video decode
   // is falling behind.
   const unsigned audioPumpThresholdMs = LOW_AUDIO_MS * 2;
 
+  // Our local low audio threshold. We may increase this if we're slow to
+  // decode video frames, in order to reduce the chance of audio underruns.
+  PRInt64 lowAudioThreshold = LOW_AUDIO_MS;
+
+  // Our local ample audio threshold. If we increase lowAudioThreshold, we'll
+  // also increase this to appropriately (we don't want lowAudioThreshold to
+  // be greater than ampleAudioThreshold, else we'd stop decoding!).
+  PRInt64 ampleAudioThreshold = AMPLE_AUDIO_MS;
+
   // Main decode loop.
   while (videoPlaying || audioPlaying) {
     PRBool audioWait = !audioPlaying;
     PRBool videoWait = !videoPlaying;
     {
       // Wait for more data to download if we've exhausted all our
       // buffered data.
       MonitorAutoEnter mon(mDecoder->GetMonitor());
@@ -232,61 +252,82 @@ void nsBuiltinDecoderStateMachine::Decod
     }
 
     // We don't want to consider skipping to the next keyframe if we've
     // only just started up the decode loop, so wait until we've decoded
     // some frames before allowing the keyframe skip.
     if (videoPump && videoQueueSize >= videoPumpThreshold) {
       videoPump = PR_FALSE;
     }
-    if (audioPlaying &&
-        !videoPump &&
-        videoPlaying &&
-        videoQueueSize < LOW_VIDEO_FRAMES)
-    {
-      skipToNextKeyframe = PR_TRUE;
-    }
 
     // Determine how much audio data is decoded ahead of the current playback
     // position.
-    PRInt64 initialDownloadPosition = 0;
     PRInt64 currentTime = 0;
     PRInt64 audioDecoded = 0;
+    PRBool decodeCloseToDownload = PR_FALSE;
     {
       MonitorAutoEnter mon(mDecoder->GetMonitor());
       currentTime = GetMediaTime();
       audioDecoded = mReader->mAudioQueue.Duration();
       if (mAudioEndTime != -1) {
         audioDecoded += mAudioEndTime - currentTime;
       }
-      initialDownloadPosition =
-        mDecoder->GetCurrentStream()->GetCachedDataEnd(mDecoder->mDecoderPosition);
+      decodeCloseToDownload = IsDecodeCloseToDownload();
     }
 
     // Don't decode any audio if the audio decode is way ahead.
-    if (audioDecoded > AMPLE_AUDIO_MS) {
+    if (audioDecoded > ampleAudioThreshold) {
       audioWait = PR_TRUE;
     }
     if (audioPump && audioDecoded > audioPumpThresholdMs) {
       audioPump = PR_FALSE;
     }
-    if (!audioPump && audioPlaying && audioDecoded < LOW_AUDIO_MS) {
+    // We'll skip the video decode to the nearest keyframe if we're low on
+    // audio, or if we're low on video, provided we're not running low on
+    // data to decode. If we're running low on downloaded data to decode,
+    // we won't start keyframe skipping, as we'll be pausing playback to buffer
+    // soon anyway and we'll want to be able to display frames immediately
+    // after buffering finishes.
+    if (!skipToNextKeyframe &&
+        videoPlaying &&
+        !decodeCloseToDownload &&
+        ((!audioPump && audioPlaying && audioDecoded < lowAudioThreshold) ||
+         (!videoPump && videoQueueSize < LOW_VIDEO_FRAMES)))
+    {
       skipToNextKeyframe = PR_TRUE;
+      LOG(PR_LOG_DEBUG, ("Skipping video decode to the next keyframe"));
     }
 
+    // Video decode.
     if (videoPlaying && !videoWait) {
+      // Time the video decode, so that if it's slow, we can increase our low
+      // audio threshold to reduce the chance of an audio underrun while we're
+      // waiting for a video decode to complete.
+      TimeStamp start = TimeStamp::Now();
       videoPlaying = mReader->DecodeVideoFrame(skipToNextKeyframe, currentTime);
+      TimeDuration decodeTime = TimeStamp::Now() - start;
+      if (!decodeCloseToDownload &&
+          THRESHOLD_FACTOR * decodeTime.ToMilliseconds() > lowAudioThreshold)
+      {
+        lowAudioThreshold =
+          NS_MIN(static_cast<PRInt64>(THRESHOLD_FACTOR * decodeTime.ToMilliseconds()),
+                 static_cast<PRInt64>(AMPLE_AUDIO_MS));
+        ampleAudioThreshold = NS_MAX(THRESHOLD_FACTOR * lowAudioThreshold,
+                                     ampleAudioThreshold);
+        LOG(PR_LOG_DEBUG,
+            ("Slow video decode, set lowAudioThreshold=%lld ampleAudioThreshold=%lld",
+             lowAudioThreshold, ampleAudioThreshold));
+      }
     }
     {
       MonitorAutoEnter mon(mDecoder->GetMonitor());
-      initialDownloadPosition =
-        mDecoder->GetCurrentStream()->GetCachedDataEnd(mDecoder->mDecoderPosition);
       mDecoder->GetMonitor().NotifyAll();
     }
 
+    // Audio decode.
     if (audioPlaying && !audioWait) {
       audioPlaying = mReader->DecodeAudioData();
     }
 
     {
       MonitorAutoEnter mon(mDecoder->GetMonitor());
 
       if (!IsPlaying()) {
@@ -840,42 +881,27 @@ PRInt64 nsBuiltinDecoderStateMachine::Au
                "Should only call AudioDecodedMs() when we have audio");
   // The amount of audio we have decoded is the amount of audio data we've
   // already decoded and pushed to the hardware, plus the amount of audio
   // data waiting to be pushed to the hardware.
   PRInt64 pushed = (mAudioEndTime != -1) ? (mAudioEndTime - GetMediaTime()) : 0;
   return pushed + mReader->mAudioQueue.Duration();
 }
 
-PRBool nsBuiltinDecoderStateMachine::HasLowDecodedData() const
+PRBool nsBuiltinDecoderStateMachine::IsDecodeCloseToDownload()
 {
-  // We consider ourselves low on decoded data if we're low on audio,
-  // provided we've not decoded to the end of the audio stream, or
-  // if we're only playing video and we're low on video frames, provided
-  // we've not decoded to the end of the video stream.
-  return ((HasAudio() &&
-           !mReader->mAudioQueue.IsFinished() &&
-           AudioDecodedMs() < LOW_AUDIO_MS)
-          ||
-         (!HasAudio() &&
-          HasVideo() &&
-          !mReader->mVideoQueue.IsFinished() &&
-          (PRUint32)mReader->mVideoQueue.GetSize() < LOW_VIDEO_FRAMES));
-}
-
-PRBool nsBuiltinDecoderStateMachine::HasAmpleDecodedData() const
-{
-  return (!HasAudio() ||
-          AudioDecodedMs() >= AMPLE_AUDIO_MS ||
-          mReader->mAudioQueue.IsFinished())
-         &&
-         (!HasVideo() ||
-          (PRUint32)mReader->mVideoQueue.GetSize() > AMPLE_VIDEO_FRAMES ||
-          mReader->mVideoQueue.AtEndOfStream());
-}
+  nsMediaStream* stream = mDecoder->GetCurrentStream();
+  PRInt64 decodePos = mDecoder->mDecoderPosition;
+  PRInt64 downloadPos = stream->GetCachedDataEnd(decodePos);
+  PRInt64 length = stream->GetLength();
+  double bufferTarget = GetDuration() / NORMAL_BUFFER_MARGIN;
+  double threshold = (bufferTarget > 0 && length != -1) ?
+    (length / (bufferTarget)) : LIVE_BUFFER_MARGIN;
+  return (downloadPos - decodePos) < threshold;
+}        
 
 nsresult nsBuiltinDecoderStateMachine::Run()
 {
   NS_ASSERTION(IsCurrentThread(mDecoder->mStateMachineThread),
                "Should be on state machine thread.");
   nsMediaStream* stream = mDecoder->GetCurrentStream();
   NS_ENSURE_TRUE(stream, NS_ERROR_NULL_POINTER);
 
@@ -956,17 +982,17 @@ nsresult nsBuiltinDecoderStateMachine::R
           continue;
         }
 
         AdvanceFrame();
 
         if (mState != DECODER_STATE_DECODING)
           continue;
 
-        if (HasLowDecodedData() &&
+        if (IsDecodeCloseToDownload() &&
             mDecoder->GetState() == nsBuiltinDecoder::PLAY_STATE_PLAYING &&
             !stream->IsDataCachedToEndOfStream(mDecoder->mDecoderPosition) &&
             !stream->IsSuspended())
         {
           // We're low on decoded data, and/or our decode has caught up with
           // the download. Let's buffer to make sure we can play a decent
           // amount of video in the future.
           StartBuffering();
@@ -1076,17 +1102,17 @@ nsresult nsBuiltinDecoderStateMachine::R
 
     case DECODER_STATE_BUFFERING:
       {
         // We will remain in the buffering state if we've not decoded enough
         // data to begin playback, or if we've not downloaded a reasonable
         // amount of data inside our buffering time.
         TimeDuration elapsed = TimeStamp::Now() - mBufferingStart;
         PRBool isLiveStream = mDecoder->GetCurrentStream()->GetLength() == -1;
-        if (((!isLiveStream && !mDecoder->CanPlayThrough()) || !HasAmpleDecodedData()) &&
+        if ((isLiveStream || !mDecoder->CanPlayThrough()) &&
              elapsed < TimeDuration::FromSeconds(BUFFERING_WAIT) &&
              stream->GetCachedDataEnd(mDecoder->mDecoderPosition) < mBufferingEndOffset &&
              !stream->IsDataCachedToEndOfStream(mDecoder->mDecoderPosition) &&
              !stream->IsSuspended())
         {
           LOG(PR_LOG_DEBUG,
               ("In buffering: buffering data until %u bytes available or %f seconds",
                PRUint32(mBufferingEndOffset - stream->GetCachedDataEnd(mDecoder->mDecoderPosition)),
--- a/content/media/nsBuiltinDecoderStateMachine.h
+++ b/content/media/nsBuiltinDecoderStateMachine.h
@@ -247,27 +247,28 @@ public:
 
   PRInt64 GetEndMediaTime() const {
     mDecoder->GetMonitor().AssertCurrentThreadIn();
     return mEndTime;
   }
 
 protected:
 
+  // Returns PR_TRUE if the decode is withing an estimated one tenth of a
+  // second's worth of data of the download, i.e. the decode has almost
+  // caught up with the download. If we can't estimate one tenth of a second's
+  // worth of data, we'll return PR_TRUE if the decode is within 100KB of
+  // the download.
+  PRBool IsDecodeCloseToDownload();
+
   // Returns the number of unplayed ms of audio we've got decoded and/or
   // pushed to the hardware waiting to play. This is how much audio we can
   // play without having to run the audio decoder.
   PRInt64 AudioDecodedMs() const;
 
-  // Returns PR_TRUE if we're running low on decoded data.
-  PRBool HasLowDecodedData() const;
-
-  // Returns PR_TRUE if we've got plenty of decoded data.
-  PRBool HasAmpleDecodedData() const;
-
   // Returns PR_TRUE when there's decoded audio waiting to play.
   // The decoder monitor must be held.
   PRBool HasFutureAudio() const;
 
   // Waits on the decoder Monitor for aMs. If the decoder monitor is awoken
   // by a Notify() call, we'll continue waiting, unless we've moved into
   // shutdown state. This enables us to ensure that we wait for a specified
   // time, and that the myriad of Notify()s we do an the decoder monitor