Bug 1194918 - Move av-sync and video frame rendering logic from MDSM to VideoSink. r=jwwang.
authorKilik Kuo <kikuo@mozilla.com>
Mon, 19 Oct 2015 18:08:11 +0800
changeset 303540 1dc6e120ebff05eb39ed64d6cbedbd72c0ab5bc9
parent 303539 00b1bb5ace0de6d802bd130f84dde0328b6f018c
child 303541 5e890d7ad1744e38b0d9de03696dfe09631c12ec
push id1001
push userraliiev@mozilla.com
push dateMon, 18 Jan 2016 19:06:03 +0000
treeherdermozilla-release@8b89261f3ac4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjwwang
bugs1194918
milestone44.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1194918 - Move av-sync and video frame rendering logic from MDSM to VideoSink. r=jwwang.
dom/media/MediaDecoderStateMachine.cpp
dom/media/MediaDecoderStateMachine.h
dom/media/mediasink/MediaSink.h
dom/media/mediasink/VideoSink.cpp
dom/media/mediasink/VideoSink.h
--- a/dom/media/MediaDecoderStateMachine.cpp
+++ b/dom/media/MediaDecoderStateMachine.cpp
@@ -198,27 +198,25 @@ static void InitVideoQueuePrefs() {
 
 MediaDecoderStateMachine::MediaDecoderStateMachine(MediaDecoder* aDecoder,
                                                    MediaDecoderReader* aReader,
                                                    bool aRealTime) :
   mDecoder(aDecoder),
   mTaskQueue(new TaskQueue(GetMediaThreadPool(MediaThreadType::PLAYBACK),
                            /* aSupportsTailDispatch = */ true)),
   mWatchManager(this, mTaskQueue),
-  mProducerID(ImageContainer::AllocateProducerID()),
   mRealTime(aRealTime),
   mDispatchedStateMachine(false),
   mDelayedScheduler(mTaskQueue),
   mState(DECODER_STATE_DECODING_NONE, "MediaDecoderStateMachine::mState"),
   mCurrentFrameID(0),
   mObservedDuration(TimeUnit(), "MediaDecoderStateMachine::mObservedDuration"),
   mFragmentEndTime(-1),
   mReader(aReader),
   mDecodedAudioEndTime(-1),
-  mVideoFrameEndTime(-1),
   mDecodedVideoEndTime(-1),
   mPlaybackRate(1.0),
   mLowAudioThresholdUsecs(detail::LOW_AUDIO_USECS),
   mAmpleAudioThresholdUsecs(detail::AMPLE_AUDIO_USECS),
   mQuickBufferingLowDataThresholdUsecs(detail::QUICK_BUFFERING_LOW_DATA_USECS),
   mIsAudioPrerolling(false),
   mIsVideoPrerolling(false),
   mAudioCaptured(false, "MediaDecoderStateMachine::mAudioCaptured"),
@@ -386,21 +384,21 @@ already_AddRefed<media::MediaSink>
 MediaDecoderStateMachine::CreateMediaSink(bool aAudioCaptured)
 {
   // TODO: We can't really create a new DecodedStream until OutputStreamManager
   //       is extracted. It is tricky that the implementation of DecodedStream
   //       happens to allow reuse after shutdown without creating a new one.
   RefPtr<media::MediaSink> audioSink = aAudioCaptured ?
     mStreamSink : CreateAudioSink();
 
-  RefPtr<media::MediaSink> mediaSink = new VideoSink(mTaskQueue,
-                                                       audioSink,
-                                                       mVideoQueue,
-                                                       mDecoder->GetVideoFrameContainer(),
-                                                       mRealTime);
+  RefPtr<media::MediaSink> mediaSink =
+    new VideoSink(mTaskQueue, audioSink, mVideoQueue,
+                  mDecoder->GetVideoFrameContainer(), mRealTime,
+                  mDecoder->GetFrameStatistics(), AUDIO_DURATION_USECS,
+                  sVideoQueueSendToCompositorSize);
   return mediaSink.forget();
 }
 
 bool MediaDecoderStateMachine::HasFutureAudio()
 {
   MOZ_ASSERT(OnTaskQueue());
   NS_ASSERTION(HasAudio(), "Should only call HasFutureAudio() when we have audio");
   // We've got audio ready to play if:
@@ -1063,17 +1061,16 @@ nsresult MediaDecoderStateMachine::Init(
 void MediaDecoderStateMachine::StopPlayback()
 {
   MOZ_ASSERT(OnTaskQueue());
   DECODER_LOG("StopPlayback()");
 
   mDecoder->DispatchPlaybackStopped();
 
   if (IsPlaying()) {
-    RenderVideoFrames(1);
     mMediaSink->SetPlaying(false);
     MOZ_ASSERT(!IsPlaying());
   }
 
   DispatchDecodeTasksIfNeeded();
 }
 
 void MediaDecoderStateMachine::MaybeStartPlayback()
@@ -1854,17 +1851,17 @@ bool MediaDecoderStateMachine::HasLowUnd
   }
 
   if (mBuffered.Ref().IsInvalid()) {
     return false;
   }
 
   int64_t endOfDecodedVideoData = INT64_MAX;
   if (HasVideo() && !VideoQueue().AtEndOfStream()) {
-    endOfDecodedVideoData = VideoQueue().Peek() ? VideoQueue().Peek()->GetEndTime() : mVideoFrameEndTime;
+    endOfDecodedVideoData = VideoQueue().Peek() ? VideoQueue().Peek()->GetEndTime() : VideoEndTime();
   }
   int64_t endOfDecodedAudioData = INT64_MAX;
   if (HasAudio() && !AudioQueue().AtEndOfStream()) {
     // mDecodedAudioEndTime could be -1 when no audio samples are decoded.
     // But that is fine since we consider ourself as low in decoded data when
     // we don't have any decoded audio samples at all.
     endOfDecodedAudioData = mDecodedAudioEndTime;
   }
@@ -2054,17 +2051,17 @@ MediaDecoderStateMachine::AdjustAudioThr
 
 void
 MediaDecoderStateMachine::FinishDecodeFirstFrame()
 {
   MOZ_ASSERT(OnTaskQueue());
   DECODER_LOG("FinishDecodeFirstFrame");
 
   if (!IsRealTime() && !mSentFirstFrameLoadedEvent) {
-    RenderVideoFrames(1);
+    mMediaSink->Redraw();
   }
 
   // If we don't know the duration by this point, we assume infinity, per spec.
   if (mDuration.Ref().isNothing()) {
     mDuration = Some(TimeUnit::FromInfinity());
   }
 
   DECODER_LOG("Media duration %lld, "
@@ -2151,17 +2148,17 @@ MediaDecoderStateMachine::SeekCompleted(
   // seek while quick-buffering, we won't bypass quick buffering mode
   // if we need to buffer after the seek.
   mQuickBuffering = false;
 
   mCurrentSeek.Resolve(mState == DECODER_STATE_COMPLETED, __func__);
   ScheduleStateMachine();
 
   if (video) {
-    RenderVideoFrames(1);
+    mMediaSink->Redraw();
     nsCOMPtr<nsIRunnable> event =
       NS_NewRunnableMethod(mDecoder, &MediaDecoder::Invalidate);
     AbstractThread::MainThread()->Dispatch(event.forget());
   }
 }
 
 class DecoderDisposer
 {
@@ -2306,17 +2303,17 @@ nsresult MediaDecoderStateMachine::RunSt
         // We're playing, but the element/decoder is in paused state. Stop
         // playing!
         StopPlayback();
       }
 
       // Start playback if necessary so that the clock can be properly queried.
       MaybeStartPlayback();
 
-      UpdateRenderedVideoFrames();
+      UpdatePlaybackPositionPeriodically();
       NS_ASSERTION(!IsPlaying() ||
                    mLogicallySeeking ||
                    IsStateMachineScheduled(),
                    "Must have timer scheduled");
       return NS_OK;
     }
 
     case DECODER_STATE_BUFFERING: {
@@ -2377,17 +2374,17 @@ nsresult MediaDecoderStateMachine::RunSt
       // once to ensure the current playback position is advanced to the
       // end of the media, and so that we update the readyState.
       if (VideoQueue().GetSize() > 1 ||
           (HasAudio() && !mAudioCompleted) ||
           (mAudioCaptured && !mStreamSink->IsFinished()))
       {
         // Start playback if necessary to play the remaining media.
         MaybeStartPlayback();
-        UpdateRenderedVideoFrames();
+        UpdatePlaybackPositionPeriodically();
         NS_ASSERTION(!IsPlaying() ||
                      mLogicallySeeking ||
                      IsStateMachineScheduled(),
                      "Must have timer scheduled");
         return NS_OK;
       }
 
       // StopPlayback in order to reset the IsPlaying() state so audio
@@ -2399,17 +2396,17 @@ nsresult MediaDecoderStateMachine::RunSt
         // our state should have scheduled another state machine run.
         NS_ASSERTION(IsStateMachineScheduled(), "Must have timer scheduled");
         return NS_OK;
       }
 
       if (mPlayState == MediaDecoder::PLAY_STATE_PLAYING &&
           !mSentPlaybackEndedEvent)
       {
-        int64_t clockTime = std::max(AudioEndTime(), mVideoFrameEndTime);
+        int64_t clockTime = std::max(AudioEndTime(), VideoEndTime());
         clockTime = std::max(int64_t(0), std::max(clockTime, Duration().ToMicroseconds()));
         UpdatePlaybackPosition(clockTime);
 
         nsCOMPtr<nsIRunnable> event =
           NS_NewRunnableMethod(mDecoder, &MediaDecoder::PlaybackEnded);
         AbstractThread::MainThread()->Dispatch(event.forget());
 
         mSentPlaybackEndedEvent = true;
@@ -2440,17 +2437,16 @@ MediaDecoderStateMachine::Reset()
              mState == DECODER_STATE_DORMANT ||
              mState == DECODER_STATE_DECODING_NONE);
 
   // Stop the audio thread. Otherwise, MediaSink might be accessing AudioQueue
   // outside of the decoder monitor while we are clearing the queue and causes
   // crash for no samples to be popped.
   StopMediaSink();
 
-  mVideoFrameEndTime = -1;
   mDecodedVideoEndTime = -1;
   mDecodedAudioEndTime = -1;
   mAudioCompleted = false;
   AudioQueue().Reset();
   VideoQueue().Reset();
   mFirstVideoFrameAfterSeek = nullptr;
   mDropAudioUntilNextDiscontinuity = true;
   mDropVideoUntilNextDiscontinuity = true;
@@ -2490,153 +2486,63 @@ MediaDecoderStateMachine::CheckFrameVali
         mCorruptFrames.clear();
       gfxCriticalNote << "Too many dropped/corrupted frames, disabling DXVA";
     }
   } else {
     mCorruptFrames.insert(0);
   }
 }
 
-void MediaDecoderStateMachine::RenderVideoFrames(int32_t aMaxFrames,
-                                                 int64_t aClockTime,
-                                                 const TimeStamp& aClockTimeStamp)
-{
-  MOZ_ASSERT(OnTaskQueue());
-
-  VideoFrameContainer* container = mDecoder->GetVideoFrameContainer();
-  nsAutoTArray<RefPtr<MediaData>,16> frames;
-  VideoQueue().GetFirstElements(aMaxFrames, &frames);
-  if (frames.IsEmpty() || !container) {
-    return;
-  }
-
-  nsAutoTArray<ImageContainer::NonOwningImage,16> images;
-  TimeStamp lastFrameTime;
-  for (uint32_t i = 0; i < frames.Length(); ++i) {
-    VideoData* frame = frames[i]->As<VideoData>();
-
-    bool valid = !frame->mImage || frame->mImage->IsValid();
-    frame->mSentToCompositor = true;
-
-    if (!valid) {
-      continue;
-    }
-
-    int64_t frameTime = frame->mTime;
-    if (frameTime < 0) {
-      // Frame times before the start time are invalid; drop such frames
-      continue;
-    }
-
-
-    TimeStamp t;
-    if (aMaxFrames > 1) {
-      MOZ_ASSERT(!aClockTimeStamp.IsNull());
-      int64_t delta = frame->mTime - aClockTime;
-      t = aClockTimeStamp +
-          TimeDuration::FromMicroseconds(delta / mPlaybackRate);
-      if (!lastFrameTime.IsNull() && t <= lastFrameTime) {
-        // Timestamps out of order; drop the new frame. In theory we should
-        // probably replace the previous frame with the new frame if the
-        // timestamps are equal, but this is a corrupt video file already so
-        // never mind.
-        continue;
-      }
-      lastFrameTime = t;
-    }
-
-    ImageContainer::NonOwningImage* img = images.AppendElement();
-    img->mTimeStamp = t;
-    img->mImage = frame->mImage;
-    img->mFrameID = frame->mFrameID;
-    img->mProducerID = mProducerID;
-
-    VERBOSE_LOG("playing video frame %lld (id=%x) (queued=%i, state-machine=%i, decoder-queued=%i)",
-                frame->mTime, frame->mFrameID,
-                VideoQueue().GetSize() + mReader->SizeOfVideoQueueInFrames(),
-                VideoQueue().GetSize(), mReader->SizeOfVideoQueueInFrames());
-  }
-
-  container->SetCurrentFrames(frames[0]->As<VideoData>()->mDisplay, images);
-}
-
 int64_t
 MediaDecoderStateMachine::GetClock(TimeStamp* aTimeStamp) const
 {
   MOZ_ASSERT(OnTaskQueue());
   int64_t clockTime = mMediaSink->GetPosition(aTimeStamp);
   NS_ASSERTION(GetMediaTime() <= clockTime, "Clock should go forwards.");
   return clockTime;
 }
 
-void MediaDecoderStateMachine::UpdateRenderedVideoFrames()
+void
+MediaDecoderStateMachine::UpdatePlaybackPositionPeriodically()
 {
   MOZ_ASSERT(OnTaskQueue());
 
   if (!IsPlaying() || mLogicallySeeking) {
     return;
   }
 
   if (mAudioCaptured) {
     DiscardStreamData();
   }
 
-  TimeStamp nowTime;
-  const int64_t clockTime = GetClock(&nowTime);
-  // Skip frames up to the frame at the playback position, and figure out
-  // the time remaining until it's time to display the next frame and drop
-  // the current frame.
-  NS_ASSERTION(clockTime >= 0, "Should have positive clock time.");
-  int64_t remainingTime = AUDIO_DURATION_USECS;
-  if (VideoQueue().GetSize() > 0) {
-    RefPtr<MediaData> currentFrame = VideoQueue().PopFront();
-    int32_t framesRemoved = 0;
-    while (VideoQueue().GetSize() > 0) {
-      MediaData* nextFrame = VideoQueue().PeekFront();
-      if (!IsRealTime() && nextFrame->mTime > clockTime) {
-        remainingTime = nextFrame->mTime - clockTime;
-        break;
-      }
-      ++framesRemoved;
-      if (!currentFrame->As<VideoData>()->mSentToCompositor) {
-        mDecoder->NotifyDecodedFrames(0, 0, 1);
-        VERBOSE_LOG("discarding video frame mTime=%lld clock_time=%lld",
-                    currentFrame->mTime, clockTime);
-      }
-      currentFrame = VideoQueue().PopFront();
-
-    }
-    VideoQueue().PushFront(currentFrame);
-    if (framesRemoved > 0) {
-      mVideoFrameEndTime = currentFrame->GetEndTime();
-      FrameStatistics& frameStats = mDecoder->GetFrameStatistics();
-      frameStats.NotifyPresentedFrame();
-    }
-  }
-
-  RenderVideoFrames(sVideoQueueSendToCompositorSize, clockTime, nowTime);
-
   // Cap the current time to the larger of the audio and video end time.
   // This ensures that if we're running off the system clock, we don't
   // advance the clock to after the media end time.
-  if (mVideoFrameEndTime != -1 || AudioEndTime() != -1) {
+  if (VideoEndTime() != -1 || AudioEndTime() != -1) {
+
+    const int64_t clockTime = GetClock();
+    // Skip frames up to the frame at the playback position, and figure out
+    // the time remaining until it's time to display the next frame and drop
+    // the current frame.
+    NS_ASSERTION(clockTime >= 0, "Should have positive clock time.");
+
     // These will be non -1 if we've displayed a video frame, or played an audio frame.
-    int64_t t = std::min(clockTime, std::max(mVideoFrameEndTime, AudioEndTime()));
+    int64_t t = std::min(clockTime, std::max(VideoEndTime(), AudioEndTime()));
     // FIXME: Bug 1091422 - chained ogg files hit this assertion.
     //MOZ_ASSERT(t >= GetMediaTime());
     if (t > GetMediaTime()) {
       UpdatePlaybackPosition(t);
     }
   }
   // Note we have to update playback position before releasing the monitor.
   // Otherwise, MediaDecoder::AddOutputStream could kick in when we are outside
   // the monitor and get a staled value from GetCurrentTimeUs() which hits the
   // assertion in GetClock().
 
-  int64_t delay = std::max<int64_t>(1, remainingTime / mPlaybackRate);
+  int64_t delay = std::max<int64_t>(1, AUDIO_DURATION_USECS / mPlaybackRate);
   ScheduleStateMachineIn(delay);
 }
 
 nsresult
 MediaDecoderStateMachine::DropVideoUpToSeekTarget(MediaData* aSample)
 {
   MOZ_ASSERT(OnTaskQueue());
   RefPtr<VideoData> video(aSample->As<VideoData>());
@@ -2929,51 +2835,65 @@ MediaDecoderStateMachine::AudioEndTime()
   MOZ_ASSERT(OnTaskQueue());
   if (mMediaSink->IsStarted()) {
     return mMediaSink->GetEndTime(TrackInfo::kAudioTrack);
   }
   MOZ_ASSERT(!HasAudio());
   return -1;
 }
 
+int64_t
+MediaDecoderStateMachine::VideoEndTime() const
+{
+  MOZ_ASSERT(OnTaskQueue());
+  if (mMediaSink->IsStarted()) {
+    return mMediaSink->GetEndTime(TrackInfo::kVideoTrack);
+  }
+  return -1;
+}
+
 void
 MediaDecoderStateMachine::OnMediaSinkVideoComplete()
 {
   MOZ_ASSERT(OnTaskQueue());
+  VERBOSE_LOG("[%s]", __func__);
 
   mMediaSinkVideoPromise.Complete();
   ScheduleStateMachine();
 }
 
 void
 MediaDecoderStateMachine::OnMediaSinkVideoError()
 {
   MOZ_ASSERT(OnTaskQueue());
+  VERBOSE_LOG("[%s]", __func__);
 
   mMediaSinkVideoPromise.Complete();
   if (HasAudio()) {
     return;
   }
   DecodeError();
 }
 
 void MediaDecoderStateMachine::OnMediaSinkAudioComplete()
 {
   MOZ_ASSERT(OnTaskQueue());
+  VERBOSE_LOG("[%s]", __func__);
 
   mMediaSinkAudioPromise.Complete();
   // Set true only when we have audio.
   mAudioCompleted = mInfo.HasAudio();
   // To notify PlaybackEnded as soon as possible.
   ScheduleStateMachine();
 }
 
 void MediaDecoderStateMachine::OnMediaSinkAudioError()
 {
   MOZ_ASSERT(OnTaskQueue());
+  VERBOSE_LOG("[%s]", __func__);
 
   mMediaSinkAudioPromise.Complete();
   // Set true only when we have audio.
   mAudioCompleted = mInfo.HasAudio();
 
   // Make the best effort to continue playback when there is video.
   if (HasVideo()) {
     return;
--- a/dom/media/MediaDecoderStateMachine.h
+++ b/dom/media/MediaDecoderStateMachine.h
@@ -123,17 +123,16 @@ extern PRLogModuleInfo* gMediaSampleLog;
 */
 class MediaDecoderStateMachine
 {
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaDecoderStateMachine)
 public:
   typedef MediaDecoderReader::AudioDataPromise AudioDataPromise;
   typedef MediaDecoderReader::VideoDataPromise VideoDataPromise;
   typedef MediaDecoderOwner::NextFrameStatus NextFrameStatus;
-  typedef mozilla::layers::ImageContainer::ProducerID ProducerID;
   typedef mozilla::layers::ImageContainer::FrameID FrameID;
   MediaDecoderStateMachine(MediaDecoder* aDecoder,
                            MediaDecoderReader* aReader,
                            bool aRealTime = false);
 
   nsresult Init();
 
   // Enumeration for the valid decoding states
@@ -460,32 +459,19 @@ protected:
   // media element -- use UpdatePlaybackPosition for that.  Called on the state
   // machine thread, caller must hold the decoder lock.
   void UpdatePlaybackPositionInternal(int64_t aTime);
 
   // Decode monitor must be held. To determine if MDSM needs to turn off HW
   // acceleration.
   void CheckFrameValidity(VideoData* aData);
 
-  // Sets VideoQueue images into the VideoFrameContainer. Called on the shared
-  // state machine thread. Decode monitor must be held. The first aMaxFrames
-  // (at most) are set.
-  // aClockTime and aClockTimeStamp are used as the baseline for deriving
-  // timestamps for the frames; when omitted, aMaxFrames must be 1 and
-  // a null timestamp is passed to the VideoFrameContainer.
-  // If the VideoQueue is empty, this does nothing.
-  void RenderVideoFrames(int32_t aMaxFrames, int64_t aClockTime = 0,
-                         const TimeStamp& aClickTimeStamp = TimeStamp());
-
-  // If we have video, display a video frame if it's time for display has
-  // arrived, otherwise sleep until it's time for the next frame. Update the
-  // current frame time as appropriate, and trigger ready state update.  The
-  // decoder monitor must be held with exactly one lock count. Called on the
-  // state machine thread.
-  void UpdateRenderedVideoFrames();
+  // Update playback position and trigger next update by default time period.
+  // Called on the state machine thread.
+  void UpdatePlaybackPositionPeriodically();
 
   media::MediaSink* CreateAudioSink();
 
   // Always create mediasink which contains an AudioSink or StreamSink inside.
   already_AddRefed<media::MediaSink> CreateMediaSink(bool aAudioCaptured);
 
   // Stops the media sink and shut it down.
   // The decoder monitor must be held with exactly one lock count.
@@ -668,20 +654,16 @@ private:
   RefPtr<MediaDecoder> mDecoder;
 
   // Task queue for running the state machine.
   RefPtr<TaskQueue> mTaskQueue;
 
   // State-watching manager.
   WatchManager<MediaDecoderStateMachine> mWatchManager;
 
-  // Producer ID to help ImageContainer distinguish different streams of
-  // FrameIDs. A unique and immutable value per MDSM.
-  const ProducerID mProducerID;
-
   // True is we are decoding a realtime stream, like a camera stream.
   const bool mRealTime;
 
   // True if we've dispatched a task to run the state machine but the task has
   // yet to run.
   bool mDispatchedStateMachine;
 
   // Used to dispatch another round schedule with specific target time.
@@ -922,24 +904,24 @@ private:
   // This is created in the state machine's constructor.
   RefPtr<MediaDecoderReader> mReader;
 
   // The end time of the last audio frame that's been pushed onto the media sink
   // in microseconds. This will approximately be the end time
   // of the audio stream, unless another frame is pushed to the hardware.
   int64_t AudioEndTime() const;
 
+  // The end time of the last rendered video frame that's been sent to
+  // compositor.
+  int64_t VideoEndTime() const;
+
   // The end time of the last decoded audio frame. This signifies the end of
   // decoded audio data. Used to check if we are low in decoded data.
   int64_t mDecodedAudioEndTime;
 
-  // The presentation end time of the last video frame which has been displayed
-  // in microseconds. Accessed from the state machine thread.
-  int64_t mVideoFrameEndTime;
-
   // The end time of the last decoded video frame. Used to check if we are low
   // on decoded video data.
   int64_t mDecodedVideoEndTime;
 
   // Playback rate. 1.0 : normal speed, 0.5 : two times slower.
   double mPlaybackRate;
 
   // Time at which we started decoding. Synchronised via decoder monitor.
--- a/dom/media/mediasink/MediaSink.h
+++ b/dom/media/mediasink/MediaSink.h
@@ -88,16 +88,21 @@ public:
   // Whether to preserve pitch of the audio track.
   // Do nothing if this sink has no audio track.
   // Can be called in any state.
   virtual void SetPreservesPitch(bool aPreservesPitch) {}
 
   // Pause/resume the playback. Only work after playback starts.
   virtual void SetPlaying(bool aPlaying) = 0;
 
+  // Single frame rendering operation may need to be done before playback
+  // started (1st frame) or right after seek completed or playback stopped.
+  // Do nothing if this sink has no video track. Can be called in any state.
+  virtual void Redraw() {};
+
   // Begin a playback session with the provided start time and media info.
   // Must be called when playback is stopped.
   virtual void Start(int64_t aStartTime, const MediaInfo& aInfo) = 0;
 
   // Finish a playback session.
   // Must be called after playback starts.
   virtual void Stop() = 0;
 
--- a/dom/media/mediasink/VideoSink.cpp
+++ b/dom/media/mediasink/VideoSink.cpp
@@ -15,168 +15,350 @@ extern PRLogModuleInfo* gMediaDecoderLog
 #define VSINK_LOG_V(msg, ...) \
   MOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, \
   ("VideoSink=%p " msg, this, ##__VA_ARGS__))
 
 using namespace mozilla::layers;
 
 namespace media {
 
+VideoSink::VideoSink(AbstractThread* aThread,
+                     MediaSink* aAudioSink,
+                     MediaQueue<MediaData>& aVideoQueue,
+                     VideoFrameContainer* aContainer,
+                     bool aRealTime,
+                     FrameStatistics& aFrameStats,
+                     int aDelayDuration,
+                     uint32_t aVQueueSentToCompositerSize)
+  : mOwnerThread(aThread)
+  , mAudioSink(aAudioSink)
+  , mVideoQueue(aVideoQueue)
+  , mContainer(aContainer)
+  , mProducerID(ImageContainer::AllocateProducerID())
+  , mRealTime(aRealTime)
+  , mFrameStats(aFrameStats)
+  , mVideoFrameEndTime(-1)
+  , mHasVideo(false)
+  , mUpdateScheduler(aThread)
+  , mDelayDuration(aDelayDuration)
+  , mVideoQueueSendToCompositorSize(aVQueueSentToCompositerSize)
+{
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+}
+
 VideoSink::~VideoSink()
 {
 }
 
 const MediaSink::PlaybackParams&
 VideoSink::GetPlaybackParams() const
 {
   AssertOwnerThread();
-  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+
   return mAudioSink->GetPlaybackParams();
 }
 
 void
 VideoSink::SetPlaybackParams(const PlaybackParams& aParams)
 {
   AssertOwnerThread();
-  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+
   mAudioSink->SetPlaybackParams(aParams);
 }
 
 RefPtr<GenericPromise>
 VideoSink::OnEnded(TrackType aType)
 {
   AssertOwnerThread();
-  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
   MOZ_ASSERT(mAudioSink->IsStarted(), "Must be called after playback starts.");
 
   if (aType == TrackInfo::kAudioTrack) {
     return mAudioSink->OnEnded(aType);
   } else if (aType == TrackInfo::kVideoTrack) {
     return mEndPromise;
   }
   return nullptr;
 }
 
 int64_t
 VideoSink::GetEndTime(TrackType aType) const
 {
   AssertOwnerThread();
-  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
   MOZ_ASSERT(mAudioSink->IsStarted(), "Must be called after playback starts.");
 
   if (aType == TrackInfo::kVideoTrack) {
     return mVideoFrameEndTime;
   } else if (aType == TrackInfo::kAudioTrack) {
     return mAudioSink->GetEndTime(aType);
   }
   return -1;
 }
 
 int64_t
 VideoSink::GetPosition(TimeStamp* aTimeStamp) const
 {
   AssertOwnerThread();
-  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
 
   return mAudioSink->GetPosition(aTimeStamp);
 }
 
 bool
 VideoSink::HasUnplayedFrames(TrackType aType) const
 {
   AssertOwnerThread();
-  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
   MOZ_ASSERT(aType == TrackInfo::kAudioTrack, "Not implemented for non audio tracks.");
 
   return mAudioSink->HasUnplayedFrames(aType);
 }
 
 void
 VideoSink::SetPlaybackRate(double aPlaybackRate)
 {
   AssertOwnerThread();
-  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
 
   mAudioSink->SetPlaybackRate(aPlaybackRate);
 }
 
 void
 VideoSink::SetPlaying(bool aPlaying)
 {
   AssertOwnerThread();
-  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
-
   VSINK_LOG_V(" playing (%d) -> (%d)", mAudioSink->IsPlaying(), aPlaying);
 
+  if (!aPlaying) {
+    // Reset any update timer if paused.
+    mUpdateScheduler.Reset();
+    // Since playback is paused, tell compositor to render only current frame.
+    RenderVideoFrames(1);
+  }
+
   mAudioSink->SetPlaying(aPlaying);
+
+  if (mHasVideo && aPlaying) {
+    // There's no thread in VideoSink for pulling video frames, need to trigger
+    // rendering while becoming playing status. because the VideoQueue may be
+    // full already.
+    TryUpdateRenderedVideoFrames();
+  }
 }
 
 void
 VideoSink::Start(int64_t aStartTime, const MediaInfo& aInfo)
 {
   AssertOwnerThread();
-  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
   VSINK_LOG("[%s]", __func__);
+
   mAudioSink->Start(aStartTime, aInfo);
 
-  if (aInfo.HasVideo()) {
+  mHasVideo = aInfo.HasVideo();
+
+  if (mHasVideo) {
     mEndPromise = mEndPromiseHolder.Ensure(__func__);
-    mVideoSinkEndRequest.Begin(mEndPromise->Then(
-      mOwnerThread.get(), __func__, this,
-      &VideoSink::OnVideoEnded,
-      &VideoSink::OnVideoEnded));
+    ConnectListener();
+    TryUpdateRenderedVideoFrames();
   }
 }
 
 void
-VideoSink::OnVideoEnded()
-{
-  AssertOwnerThread();
-
-  mVideoSinkEndRequest.Complete();
-}
-
-void
 VideoSink::Stop()
 {
   AssertOwnerThread();
-  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
   MOZ_ASSERT(mAudioSink->IsStarted(), "playback not started.");
   VSINK_LOG("[%s]", __func__);
 
   mAudioSink->Stop();
 
-  mVideoSinkEndRequest.DisconnectIfExists();
-  mEndPromiseHolder.ResolveIfExists(true, __func__);
-  mEndPromise = nullptr;
+  mUpdateScheduler.Reset();
+  if (mHasVideo) {
+    DisconnectListener();
+    mEndPromiseHolder.Resolve(true, __func__);
+    mEndPromise = nullptr;
+  }
+  mVideoFrameEndTime = -1;
 }
 
 bool
 VideoSink::IsStarted() const
 {
   AssertOwnerThread();
-  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
 
   return mAudioSink->IsStarted();
 }
 
 bool
 VideoSink::IsPlaying() const
 {
   AssertOwnerThread();
-  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
 
   return mAudioSink->IsPlaying();
 }
 
 void
 VideoSink::Shutdown()
 {
   AssertOwnerThread();
-  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
   MOZ_ASSERT(!mAudioSink->IsStarted(), "must be called after playback stops.");
   VSINK_LOG("[%s]", __func__);
 
   mAudioSink->Shutdown();
 }
 
+void
+VideoSink::OnVideoQueueEvent()
+{
+  AssertOwnerThread();
+  // Listen to push event, VideoSink should try rendering ASAP if first frame
+  // arrives but update scheduler is not triggered yet.
+  TryUpdateRenderedVideoFrames();
+}
+
+void
+VideoSink::Redraw()
+{
+  AssertOwnerThread();
+  RenderVideoFrames(1);
+}
+
+void
+VideoSink::TryUpdateRenderedVideoFrames()
+{
+  AssertOwnerThread();
+  if (!mUpdateScheduler.IsScheduled() && VideoQueue().GetSize() >= 1 &&
+      mAudioSink->IsPlaying()) {
+    UpdateRenderedVideoFrames();
+  }
+}
+
+void
+VideoSink::UpdateRenderedVideoFramesByTimer()
+{
+  AssertOwnerThread();
+  mUpdateScheduler.CompleteRequest();
+  UpdateRenderedVideoFrames();
+}
+
+void
+VideoSink::ConnectListener()
+{
+  AssertOwnerThread();
+  mPushListener = VideoQueue().PushEvent().Connect(
+    mOwnerThread, this, &VideoSink::OnVideoQueueEvent);
+}
+
+void
+VideoSink::DisconnectListener()
+{
+  AssertOwnerThread();
+  mPushListener.Disconnect();
+}
+
+void
+VideoSink::RenderVideoFrames(int32_t aMaxFrames,
+                             int64_t aClockTime,
+                             const TimeStamp& aClockTimeStamp)
+{
+  AssertOwnerThread();
+
+  nsAutoTArray<RefPtr<MediaData>,16> frames;
+  VideoQueue().GetFirstElements(aMaxFrames, &frames);
+  if (frames.IsEmpty() || !mContainer) {
+    return;
+  }
+
+  nsAutoTArray<ImageContainer::NonOwningImage,16> images;
+  TimeStamp lastFrameTime;
+  MediaSink::PlaybackParams params = mAudioSink->GetPlaybackParams();
+  for (uint32_t i = 0; i < frames.Length(); ++i) {
+    VideoData* frame = frames[i]->As<VideoData>();
+
+    bool valid = !frame->mImage || frame->mImage->IsValid();
+    frame->mSentToCompositor = true;
+
+    if (!valid) {
+      continue;
+    }
+
+    int64_t frameTime = frame->mTime;
+    if (frameTime < 0) {
+      // Frame times before the start time are invalid; drop such frames
+      continue;
+    }
+
+    TimeStamp t;
+    if (aMaxFrames > 1) {
+      MOZ_ASSERT(!aClockTimeStamp.IsNull());
+      int64_t delta = frame->mTime - aClockTime;
+      t = aClockTimeStamp +
+          TimeDuration::FromMicroseconds(delta / params.mPlaybackRate);
+      if (!lastFrameTime.IsNull() && t <= lastFrameTime) {
+        // Timestamps out of order; drop the new frame. In theory we should
+        // probably replace the previous frame with the new frame if the
+        // timestamps are equal, but this is a corrupt video file already so
+        // never mind.
+        continue;
+      }
+      lastFrameTime = t;
+    }
+
+    ImageContainer::NonOwningImage* img = images.AppendElement();
+    img->mTimeStamp = t;
+    img->mImage = frame->mImage;
+    img->mFrameID = frame->mFrameID;
+    img->mProducerID = mProducerID;
+
+    VSINK_LOG_V("playing video frame %lld (id=%x) (vq-queued=%i)",
+                frame->mTime, frame->mFrameID, VideoQueue().GetSize());
+  }
+  mContainer->SetCurrentFrames(frames[0]->As<VideoData>()->mDisplay, images);
+}
+
+void
+VideoSink::UpdateRenderedVideoFrames()
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink->IsPlaying(), "should be called while playing.");
+
+  TimeStamp nowTime;
+  const int64_t clockTime = mAudioSink->GetPosition(&nowTime);
+  // Skip frames up to the frame at the playback position, and figure out
+  // the time remaining until it's time to display the next frame and drop
+  // the current frame.
+  NS_ASSERTION(clockTime >= 0, "Should have positive clock time.");
+
+  int64_t remainingTime = mDelayDuration;
+  if (VideoQueue().GetSize() > 0) {
+    RefPtr<MediaData> currentFrame = VideoQueue().PopFront();
+    int32_t framesRemoved = 0;
+    while (VideoQueue().GetSize() > 0) {
+      MediaData* nextFrame = VideoQueue().PeekFront();
+      if (!mRealTime && nextFrame->mTime > clockTime) {
+        remainingTime = nextFrame->mTime - clockTime;
+        break;
+      }
+      ++framesRemoved;
+      if (!currentFrame->As<VideoData>()->mSentToCompositor) {
+        mFrameStats.NotifyDecodedFrames(0, 0, 1);
+        VSINK_LOG_V("discarding video frame mTime=%lld clock_time=%lld",
+                    currentFrame->mTime, clockTime);
+      }
+      currentFrame = VideoQueue().PopFront();
+    }
+    VideoQueue().PushFront(currentFrame);
+    if (framesRemoved > 0) {
+      mVideoFrameEndTime = currentFrame->GetEndTime();
+      mFrameStats.NotifyPresentedFrame();
+    }
+  }
+
+  RenderVideoFrames(mVideoQueueSendToCompositorSize, clockTime, nowTime);
+
+  TimeStamp target = nowTime + TimeDuration::FromMicroseconds(remainingTime);
+
+  RefPtr<VideoSink> self = this;
+  mUpdateScheduler.Ensure(target, [self] () {
+    self->UpdateRenderedVideoFramesByTimer();
+  }, [self] () {
+    self->UpdateRenderedVideoFramesByTimer();
+  });
+}
+
 } // namespace media
 } // namespace mozilla
--- a/dom/media/mediasink/VideoSink.h
+++ b/dom/media/mediasink/VideoSink.h
@@ -2,46 +2,46 @@
 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef VideoSink_h_
 #define VideoSink_h_
 
+#include "FrameStatistics.h"
 #include "ImageContainer.h"
+#include "MediaEventSource.h"
 #include "MediaSink.h"
+#include "MediaTimer.h"
 #include "mozilla/AbstractThread.h"
 #include "mozilla/MozPromise.h"
 #include "mozilla/RefPtr.h"
 #include "mozilla/TimeStamp.h"
 #include "VideoFrameContainer.h"
 
 namespace mozilla {
 
 class VideoFrameContainer;
 template <class T> class MediaQueue;
 
 namespace media {
 
 class VideoSink : public MediaSink
 {
+  typedef mozilla::layers::ImageContainer::ProducerID ProducerID;
 public:
   VideoSink(AbstractThread* aThread,
             MediaSink* aAudioSink,
             MediaQueue<MediaData>& aVideoQueue,
             VideoFrameContainer* aContainer,
-            bool aRealTime)
-    : mOwnerThread(aThread)
-    , mAudioSink(aAudioSink)
-    , mVideoQueue(aVideoQueue)
-    , mContainer(aContainer)
-    , mRealTime(aRealTime)
-    , mVideoFrameEndTime(-1)
-  {}
+            bool aRealTime,
+            FrameStatistics& aFrameStats,
+            int aDelayDuration,
+            uint32_t aVQueueSentToCompositerSize);
 
   const PlaybackParams& GetPlaybackParams() const override;
 
   void SetPlaybackParams(const PlaybackParams& aParams) override;
 
   RefPtr<GenericPromise> OnEnded(TrackType aType) override;
 
   int64_t GetEndTime(TrackType aType) const override;
@@ -49,53 +49,102 @@ public:
   int64_t GetPosition(TimeStamp* aTimeStamp = nullptr) const override;
 
   bool HasUnplayedFrames(TrackType aType) const override;
 
   void SetPlaybackRate(double aPlaybackRate) override;
 
   void SetPlaying(bool aPlaying) override;
 
+  void Redraw() override;
+
   void Start(int64_t aStartTime, const MediaInfo& aInfo) override;
 
   void Stop() override;
 
   bool IsStarted() const override;
 
   bool IsPlaying() const override;
 
   void Shutdown() override;
 
 private:
   virtual ~VideoSink();
 
+  // VideoQueue listener related.
+  void OnVideoQueueEvent();
+  void ConnectListener();
+  void DisconnectListener();
+
+  // Sets VideoQueue images into the VideoFrameContainer. Called on the shared
+  // state machine thread. The first aMaxFrames (at most) are set.
+  // aClockTime and aClockTimeStamp are used as the baseline for deriving
+  // timestamps for the frames; when omitted, aMaxFrames must be 1 and
+  // a null timestamp is passed to the VideoFrameContainer.
+  // If the VideoQueue is empty, this does nothing.
+  void RenderVideoFrames(int32_t aMaxFrames, int64_t aClockTime = 0,
+                         const TimeStamp& aClickTimeStamp = TimeStamp());
+
+  // Triggered while videosink is started, videosink becomes "playing" status,
+  // or VideoQueue event arrived.
+  void TryUpdateRenderedVideoFrames();
+
+  // If we have video, display a video frame if it's time for display has
+  // arrived, otherwise sleep until it's time for the next frame. Update the
+  // current frame time as appropriate, and trigger ready state update.
+  // Called on the shared state machine thread.
+  void UpdateRenderedVideoFrames();
+  void UpdateRenderedVideoFramesByTimer();
+
   void AssertOwnerThread() const
   {
     MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
   }
 
   MediaQueue<MediaData>& VideoQueue() const {
     return mVideoQueue;
   }
 
-  void OnVideoEnded();
-
   const RefPtr<AbstractThread> mOwnerThread;
   RefPtr<MediaSink> mAudioSink;
   MediaQueue<MediaData>& mVideoQueue;
   VideoFrameContainer* mContainer;
 
+  // Producer ID to help ImageContainer distinguish different streams of
+  // FrameIDs. A unique and immutable value per VideoSink.
+  const ProducerID mProducerID;
+
   // True if we are decoding a real-time stream.
   const bool mRealTime;
 
+  // Used to notify MediaDecoder's frame statistics
+  FrameStatistics& mFrameStats;
+
   RefPtr<GenericPromise> mEndPromise;
   MozPromiseHolder<GenericPromise> mEndPromiseHolder;
   MozPromiseRequestHolder<GenericPromise> mVideoSinkEndRequest;
 
   // The presentation end time of the last video frame which has been displayed
   // in microseconds.
   int64_t mVideoFrameEndTime;
+
+  // Event listeners for VideoQueue
+  MediaEventListener mPushListener;
+
+  // True if this sink is going to handle video track.
+  bool mHasVideo;
+
+  // Used to trigger another update of rendered frames in next round.
+  DelayedScheduler mUpdateScheduler;
+
+  // A delay duration to trigger next time UpdateRenderedVideoFrames().
+  // Based on the default value in MDSM.
+  const int mDelayDuration;
+
+  // Max frame number sent to compositor at a time.
+  // Based on the pref value obtained in MDSM.
+  const uint32_t mVideoQueueSendToCompositorSize;
 };
 
 } // namespace media
 } // namespace mozilla
 
 #endif