Bug 1143575. Keep currently-rendered frame at the front of the video queue. r=cpearce draft
authorRobert O'Callahan <robert@ocallahan.org>
Fri, 03 Jul 2015 19:29:30 +1200
changeset 275575 f57771ffd0f6c01a6dda19be33d8366fffc167c8
parent 275574 5b07b595a5ca7206f56dfae421c6b8a7230e899d
child 275576 439bfa84977b11c6b1cfd963be54bb7b865c90c2
push id3189
push userrocallahan@mozilla.com
push dateFri, 03 Jul 2015 11:12:01 +0000
reviewerscpearce
bugs1143575
milestone42.0a1
Bug 1143575. Keep currently-rendered frame at the front of the video queue. r=cpearce This makes normal playback consistent with the buffering state, which already does this. We'll also need this when we handle multiple images, because then we need to hande the entire queue of images to the ImageContainer without pulling any of them off the queue.
dom/media/MediaData.cpp
dom/media/MediaData.h
dom/media/MediaDecoderStateMachine.cpp
dom/media/MediaDecoderStateMachine.h
--- a/dom/media/MediaData.cpp
+++ b/dom/media/MediaData.cpp
@@ -111,16 +111,17 @@ IsInEmulator()
 VideoData::VideoData(int64_t aOffset,
                      int64_t aTime,
                      int64_t aDuration,
                      bool aKeyframe,
                      int64_t aTimecode,
                      IntSize aDisplay)
   : MediaData(VIDEO_DATA, aOffset, aTime, aDuration)
   , mDisplay(aDisplay)
+  , mSentToCompositor(false)
 {
   NS_ASSERTION(mDuration >= 0, "Frame must have non-negative duration.");
   mKeyframe = aKeyframe;
   mTimecode = aTimecode;
 }
 
 VideoData::~VideoData()
 {
--- a/dom/media/MediaData.h
+++ b/dom/media/MediaData.h
@@ -275,16 +275,18 @@ public:
   // will be scaled to this size. This is should be the picture region's
   // dimensions scaled with respect to its aspect ratio.
   const IntSize mDisplay;
 
   // This frame's image.
   nsRefPtr<Image> mImage;
 
 
+  bool mSentToCompositor;
+
   VideoData(int64_t aOffset,
             int64_t aTime,
             int64_t aDuration,
             bool aKeyframe,
             int64_t aTimecode,
             IntSize aDisplay);
 
 protected:
--- a/dom/media/MediaDecoderStateMachine.cpp
+++ b/dom/media/MediaDecoderStateMachine.cpp
@@ -95,17 +95,17 @@ const int64_t AMPLE_AUDIO_USECS = 100000
 // the amount of decoded audio we buffer, reducing our memory usage. We only
 // need to decode far ahead when we're decoding video using software decoding,
 // as otherwise a long video decode could cause an audio underrun.
 const int64_t NO_VIDEO_AMPLE_AUDIO_DIVISOR = 8;
 
 // If we have fewer than LOW_VIDEO_FRAMES decoded frames, and
 // we're not "prerolling video", we'll skip the video up to the next keyframe
 // which is at or after the current playback position.
-static const uint32_t LOW_VIDEO_FRAMES = 1;
+static const uint32_t LOW_VIDEO_FRAMES = 2;
 
 // Threshold in usecs that used to check if we are low on decoded video.
 // If the last video frame's end time |mDecodedVideoEndTime| is more than
 // |LOW_VIDEO_THRESHOLD_USECS*mPlaybackRate| after the current clock in
 // Advanceframe(), the video decode is lagging, and we skip to next keyframe.
 static const int32_t LOW_VIDEO_THRESHOLD_USECS = 60000;
 
 // Arbitrary "frame duration" when playing only audio.
@@ -338,17 +338,17 @@ bool MediaDecoderStateMachine::HasFuture
           AudioQueue().IsFinished());
 }
 
 bool MediaDecoderStateMachine::HaveNextFrameData()
 {
   MOZ_ASSERT(OnTaskQueue());
   AssertCurrentThreadInMonitor();
   return (!HasAudio() || HasFutureAudio()) &&
-         (!HasVideo() || VideoQueue().GetSize() > 0);
+         (!HasVideo() || VideoQueue().GetSize() > 1);
 }
 
 int64_t MediaDecoderStateMachine::GetDecodedAudioDuration()
 {
   MOZ_ASSERT(OnTaskQueue());
   AssertCurrentThreadInMonitor();
   int64_t audioDecoded = AudioQueue().Duration();
   if (mAudioEndTime != -1) {
@@ -637,17 +637,17 @@ bool MediaDecoderStateMachine::HaveEnoug
   return true;
 }
 
 bool MediaDecoderStateMachine::HaveEnoughDecodedVideo()
 {
   MOZ_ASSERT(OnTaskQueue());
   AssertCurrentThreadInMonitor();
 
-  if (static_cast<uint32_t>(VideoQueue().GetSize()) < GetAmpleVideoFrames() * mPlaybackRate) {
+  if (VideoQueue().GetSize() - 1 < GetAmpleVideoFrames() * mPlaybackRate) {
     return false;
   }
 
   DecodedStreamData* stream = GetDecodedStream();
 
   if (stream && stream->mStreamInitialized && !stream->mHaveSentFinishVideo) {
     MOZ_ASSERT(mInfo.HasVideo());
     TrackID videoTrackId = mInfo.mVideo.mTrackId;
@@ -2387,17 +2387,17 @@ MediaDecoderStateMachine::SeekCompleted(
   bool isLiveStream = mDecoder->GetResource()->IsLiveStream();
   if (mPendingSeek.Exists()) {
     // A new seek target came in while we were processing the old one. No rest
     // for the seeking.
     DECODER_LOG("A new seek came along while we were finishing the old one - staying in SEEKING");
     SetState(DECODER_STATE_SEEKING);
   } else if (GetMediaTime() == Duration().ToMicroseconds() && !isLiveStream) {
     // Seeked to end of media, move to COMPLETED state. Note we don't do
-    // this if we're playing a live stream, since the end of media will advance
+    // this when playing a live stream, since the end of media will advance
     // once we download more data!
     DECODER_LOG("Changed state from SEEKING (to %lld) to COMPLETED", seekTime);
     // Explicitly set our state so we don't decode further, and so
     // we report playback ended to the media element.
     SetState(DECODER_STATE_COMPLETED);
     DispatchDecodeTasksIfNeeded();
   } else {
     DECODER_LOG("Changed state from SEEKING (to %lld) to DECODING", seekTime);
@@ -2620,17 +2620,17 @@ nsresult MediaDecoderStateMachine::RunSt
 
     case DECODER_STATE_COMPLETED: {
       if (mPlayState != MediaDecoder::PLAY_STATE_PLAYING && IsPlaying()) {
         StopPlayback();
       }
       // Play the remaining media. We want to run AdvanceFrame() at least
       // once to ensure the current playback position is advanced to the
       // end of the media, and so that we update the readyState.
-      if (VideoQueue().GetSize() > 0 ||
+      if (VideoQueue().GetSize() > 1 ||
           (HasAudio() && !mAudioCompleted) ||
           (mAudioCaptured && !GetDecodedStream()->IsFinished()))
       {
         // Start playback if necessary to play the remaining media.
         MaybeStartPlayback();
         UpdateRenderedVideoFrames();
         NS_ASSERTION(!IsPlaying() ||
                      mLogicallySeeking ||
@@ -2745,16 +2745,17 @@ void MediaDecoderStateMachine::RenderVid
         nsCOMPtr<nsIRunnable> task =
           NS_NewRunnableMethod(mReader, &MediaDecoderReader::DisableHardwareAcceleration);
         DecodeTaskQueue()->Dispatch(task.forget());
         mDisabledHardwareAcceleration = true;
       }
     } else {
       mCorruptFrames.insert(0);
     }
+    aData->mSentToCompositor = true;
     container->SetCurrentFrame(aData->mDisplay, aData->mImage, aTarget);
     MOZ_ASSERT(container->GetFrameDelay() >= 0 || IsRealTime());
   }
 }
 
 void MediaDecoderStateMachine::ResyncAudioClock()
 {
   MOZ_ASSERT(OnTaskQueue());
@@ -2847,41 +2848,42 @@ void MediaDecoderStateMachine::UpdateRen
 
   if (mAudioCaptured) {
     SendStreamData();
   }
 
   TimeStamp nowTime;
   const int64_t clockTime = GetClock(&nowTime);
   // Skip frames up to the frame at the playback position, and figure out
-  // the time remaining until it's time to display the next frame.
+  // the time remaining until it's time to display the next frame and drop
+  // the current frame.
+  NS_ASSERTION(clockTime >= 0, "Should have positive clock time.");
   int64_t remainingTime = AUDIO_DURATION_USECS;
-  NS_ASSERTION(clockTime >= 0, "Should have positive clock time.");
   nsRefPtr<VideoData> currentFrame;
   if (VideoQueue().GetSize() > 0) {
-    VideoData* frame = VideoQueue().PeekFront();
-    int32_t droppedFrames = 0;
-    while (IsRealTime() || clockTime >= frame->mTime) {
-      mVideoFrameEndTime = frame->GetEndTime();
-      if (currentFrame) {
-        mDecoder->NotifyDecodedFrames(0, 0, 1);
-        VERBOSE_LOG("discarding video frame mTime=%lld clock_time=%lld (%d so far)",
-                    currentFrame->mTime, clockTime, ++droppedFrames);
+    currentFrame = VideoQueue().PopFront();
+    int32_t framesRemoved = 0;
+    while (VideoQueue().GetSize() > 0) {
+      VideoData* nextFrame = VideoQueue().PeekFront();
+      if (!IsRealTime() && nextFrame->mTime > clockTime) {
+        remainingTime = nextFrame->mTime - clockTime;
+        break;
       }
-      currentFrame = frame;
-      nsRefPtr<VideoData> releaseMe = VideoQueue().PopFront();
-      OnPlaybackOffsetUpdate(frame->mOffset);
-      if (VideoQueue().GetSize() == 0)
-        break;
-      frame = VideoQueue().PeekFront();
+      ++framesRemoved;
+      if (!currentFrame->mSentToCompositor) {
+        mDecoder->NotifyDecodedFrames(0, 0, 1);
+        VERBOSE_LOG("discarding video frame mTime=%lld clock_time=%lld",
+                    currentFrame->mTime, clockTime);
+      }
+      currentFrame = VideoQueue().PopFront();
     }
-    // Current frame has already been presented, wait until it's time to
-    // present the next frame.
-    if (frame && !currentFrame) {
-      remainingTime = frame->mTime - clockTime;
+    VideoQueue().PushFront(currentFrame);
+    if (framesRemoved > 0) {
+      OnPlaybackOffsetUpdate(currentFrame->mOffset);
+      mVideoFrameEndTime = currentFrame->GetEndTime();
     }
   }
 
   // Check to see if we don't have enough data to play up to the next frame.
   // If we don't, switch to buffering mode.
   if (mState == DECODER_STATE_DECODING &&
       mPlayState == MediaDecoder::PLAY_STATE_PLAYING &&
       mDecoder->IsExpectingMoreData()) {
@@ -2890,19 +2892,16 @@ void MediaDecoderStateMachine::UpdateRen
       shouldBuffer = HasLowDecodedData(remainingTime + EXHAUSTED_DATA_MARGIN_USECS) &&
                      (JustExitedQuickBuffering() || HasLowUndecodedData());
     } else {
       MOZ_ASSERT(mReader->IsWaitForDataSupported());
       shouldBuffer = (OutOfDecodedAudio() && mAudioWaitRequest.Exists()) ||
                      (OutOfDecodedVideo() && mVideoWaitRequest.Exists());
     }
     if (shouldBuffer) {
-      if (currentFrame) {
-        PushFront(currentFrame);
-      }
       StartBuffering();
       // Don't go straight back to the state machine loop since that might
       // cause us to start decoding again and we could flip-flop between
       // decoding and quick-buffering.
       ScheduleStateMachineIn(USECS_PER_S);
       return;
     }
   }
@@ -2937,38 +2936,18 @@ void MediaDecoderStateMachine::UpdateRen
     }
     MOZ_ASSERT(IsPlaying());
     MediaDecoder::FrameStatistics& frameStats = mDecoder->GetFrameStatistics();
     frameStats.NotifyPresentedFrame();
     remainingTime = currentFrame->GetEndTime() - clockTime;
     currentFrame = nullptr;
   }
 
-  // The remainingTime is negative (include zero):
-  // 1. When the clockTime is larger than the latest video frame's endtime.
-  // All the video frames should be rendered or dropped, nothing left in
-  // VideoQueue. And since the VideoQueue is empty, we don't need to wake up
-  // statemachine thread immediately, so set the remainingTime to default value.
-  // 2. Current frame's endtime is smaller than clockTime but there still exist
-  // newer frames in queue. Re-calculate the remainingTime.
-  if (remainingTime <= 0) {
-    VideoData* nextFrame = VideoQueue().PeekFront();
-    if (nextFrame) {
-      remainingTime = nextFrame->mTime - clockTime;
-    } else {
-      remainingTime = AUDIO_DURATION_USECS;
-    }
-  }
-
-  int64_t delay = remainingTime / mPlaybackRate;
-  if (delay > 0) {
-    ScheduleStateMachineIn(delay);
-  } else {
-    ScheduleStateMachine();
-  }
+  int64_t delay = std::max<int64_t>(1, remainingTime / mPlaybackRate);
+  ScheduleStateMachineIn(delay);
 }
 
 nsresult
 MediaDecoderStateMachine::DropVideoUpToSeekTarget(VideoData* aSample)
 {
   MOZ_ASSERT(OnTaskQueue());
   nsRefPtr<VideoData> video(aSample);
   MOZ_ASSERT(video);
--- a/dom/media/MediaDecoderStateMachine.h
+++ b/dom/media/MediaDecoderStateMachine.h
@@ -428,19 +428,17 @@ protected:
   // May not be invoked when mReader->UseBufferingHeuristics() is false.
   bool HasLowDecodedData(int64_t aAudioUsecs);
 
   bool OutOfDecodedAudio();
 
   bool OutOfDecodedVideo()
   {
     MOZ_ASSERT(OnTaskQueue());
-    // In buffering mode, we keep the last already-played frame in the queue.
-    int emptyVideoSize = mState == DECODER_STATE_BUFFERING ? 1 : 0;
-    return IsVideoDecoding() && !VideoQueue().IsFinished() && VideoQueue().GetSize() <= emptyVideoSize;
+    return IsVideoDecoding() && !VideoQueue().IsFinished() && VideoQueue().GetSize() <= 1;
   }
 
 
   // Returns true if we're running low on data which is not yet decoded.
   // The decoder monitor must be held.
   bool HasLowUndecodedData();
 
   // Returns true if we have less than aUsecs of undecoded data available.
@@ -1160,25 +1158,27 @@ protected:
     MOZ_ASSERT(OnTaskQueue());
     return IsRealTime() ? 0 : GetAmpleVideoFrames() / 2;
   }
 
   bool DonePrerollingAudio()
   {
     MOZ_ASSERT(OnTaskQueue());
     AssertCurrentThreadInMonitor();
-    return !IsAudioDecoding() || GetDecodedAudioDuration() >= AudioPrerollUsecs() * mPlaybackRate;
+    return !IsAudioDecoding() ||
+        GetDecodedAudioDuration() >= AudioPrerollUsecs() * mPlaybackRate;
   }
 
   bool DonePrerollingVideo()
   {
     MOZ_ASSERT(OnTaskQueue());
     AssertCurrentThreadInMonitor();
     return !IsVideoDecoding() ||
-           static_cast<uint32_t>(VideoQueue().GetSize()) >= VideoPrerollFrames() * mPlaybackRate;
+        static_cast<uint32_t>(VideoQueue().GetSize()) >=
+            VideoPrerollFrames() * mPlaybackRate + 1;
   }
 
   void StopPrerollingAudio()
   {
     MOZ_ASSERT(OnTaskQueue());
     AssertCurrentThreadInMonitor();
     if (mIsAudioPrerolling) {
       mIsAudioPrerolling = false;