Bug 1521964 - Allow VideoSink to have a secondary VideoFrameContainer assigned to it. r=jya
authorMike Conley <mconley@mozilla.com>
Fri, 01 Mar 2019 22:36:33 +0000
changeset 519920 b7a51e2d1d66f6cc8a9792741dd911014a9e0db3
parent 519919 fbfa1fe0e12ac71bdc9869561162d74f4ce9c978
child 519921 45807d96ca7f5378567115595b98db0d2f3cd0bc
push id10862
push userffxbld-merge
push dateMon, 11 Mar 2019 13:01:11 +0000
treeherdermozilla-beta@a2e7f5c935da [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjya
bugs1521964
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1521964 - Allow VideoSink to have a secondary VideoFrameContainer assigned to it. r=jya Differential Revision: https://phabricator.services.mozilla.com/D20022
dom/media/MediaDecoderStateMachine.cpp
dom/media/MediaDecoderStateMachine.h
dom/media/mediasink/MediaSink.h
dom/media/mediasink/VideoSink.cpp
dom/media/mediasink/VideoSink.h
--- a/dom/media/MediaDecoderStateMachine.cpp
+++ b/dom/media/MediaDecoderStateMachine.cpp
@@ -3551,16 +3551,27 @@ RefPtr<GenericPromise> MediaDecoderState
     nsresult rv = StartMediaSink();
     if (NS_FAILED(rv)) {
       return GenericPromise::CreateAndReject(NS_ERROR_ABORT, __func__);
     }
   }
   return GenericPromise::CreateAndResolve(wasPlaying, __func__);
 }
 
+void MediaDecoderStateMachine::SetSecondaryVideoContainer(
+    const RefPtr<VideoFrameContainer>& aSecondary) {
+  MOZ_ASSERT(NS_IsMainThread());
+
+  RefPtr<MediaDecoderStateMachine> self = this;
+  Unused << InvokeAsync(OwnerThread(), __func__, [self, aSecondary]() {
+    self->mMediaSink->SetSecondaryVideoContainer(aSecondary);
+    return GenericPromise::CreateAndResolve(true, __func__);
+  });
+}
+
 TimeUnit MediaDecoderStateMachine::AudioEndTime() const {
   MOZ_ASSERT(OnTaskQueue());
   if (mMediaSink->IsStarted()) {
     return mMediaSink->GetEndTime(TrackInfo::kAudioTrack);
   }
   return GetMediaTime();
 }
 
--- a/dom/media/MediaDecoderStateMachine.h
+++ b/dom/media/MediaDecoderStateMachine.h
@@ -283,16 +283,19 @@ class MediaDecoderStateMachine
 
   size_t SizeOfAudioQueue() const;
 
   // Sets the video decode mode. Used by the suspend-video-decoder feature.
   void SetVideoDecodeMode(VideoDecodeMode aMode);
 
   RefPtr<GenericPromise> InvokeSetSink(RefPtr<AudioDeviceInfo> aSink);
 
+  void SetSecondaryVideoContainer(
+      const RefPtr<VideoFrameContainer>& aSecondary);
+
  private:
   class StateObject;
   class DecodeMetadataState;
   class DormantState;
   class DecodingFirstFrameState;
   class DecodingState;
   class LoopingDecodingState;
   class SeekingState;
--- a/dom/media/mediasink/MediaSink.h
+++ b/dom/media/mediasink/MediaSink.h
@@ -11,16 +11,17 @@
 #include "MediaInfo.h"
 #include "mozilla/MozPromise.h"
 #include "mozilla/RefPtr.h"
 #include "nsISupportsImpl.h"
 
 namespace mozilla {
 
 class TimeStamp;
+class VideoFrameContainer;
 
 /**
  * A consumer of audio/video data which plays audio and video tracks and
  * manages A/V sync between them.
  *
  * A typical sink sends audio/video outputs to the speaker and screen.
  * However, there are also sinks which capture the output of an media element
  * and send the output to a MediaStream.
@@ -120,16 +121,19 @@ class MediaSink {
   // Can be called in any state.
   virtual bool IsPlaying() const = 0;
 
   // Called on the state machine thread to shut down the sink. All resources
   // allocated by this sink should be released.
   // Must be called after playback stopped.
   virtual void Shutdown() {}
 
+  virtual void SetSecondaryVideoContainer(VideoFrameContainer* aSecondary) {}
+  virtual void ClearSecondaryVideoContainer() {}
+
   // Return a string containing debugging information.
   // Can be called in any phase.
   virtual nsCString GetDebugInfo() { return nsCString(); }
 
  protected:
   virtual ~MediaSink() = default;
 };
 
--- a/dom/media/mediasink/VideoSink.cpp
+++ b/dom/media/mediasink/VideoSink.cpp
@@ -166,16 +166,19 @@ void VideoSink::SetPlaying(bool aPlaying
   if (!aPlaying) {
     // Reset any update timer if paused.
     mUpdateScheduler.Reset();
     // Since playback is paused, tell compositor to render only current frame.
     RenderVideoFrames(1);
     if (mContainer) {
       mContainer->ClearCachedResources();
     }
+    if (mSecondaryContainer) {
+      mSecondaryContainer->ClearCachedResources();
+    }
   }
 
   mAudioSink->SetPlaying(aPlaying);
 
   if (mHasVideo && aPlaying) {
     // There's no thread in VideoSink for pulling video frames, need to trigger
     // rendering while becoming playing status. because the VideoQueue may be
     // full already.
@@ -293,30 +296,39 @@ void VideoSink::OnVideoQueueFinished() {
 void VideoSink::Redraw(const VideoInfo& aInfo) {
   AssertOwnerThread();
 
   // No video track, nothing to draw.
   if (!aInfo.IsValid() || !mContainer) {
     return;
   }
 
+  auto now = TimeStamp::Now();
+
   RefPtr<VideoData> video = VideoQueue().PeekFront();
   if (video) {
     video->MarkSentToCompositor();
-    mContainer->SetCurrentFrame(video->mDisplay, video->mImage,
-                                TimeStamp::Now());
+    mContainer->SetCurrentFrame(video->mDisplay, video->mImage, now);
+    if (mSecondaryContainer) {
+      mSecondaryContainer->SetCurrentFrame(video->mDisplay, video->mImage, now);
+    }
     return;
   }
 
   // When we reach here, it means there are no frames in this video track.
   // Draw a blank frame to ensure there is something in the image container
   // to fire 'loadeddata'.
+
   RefPtr<Image> blank =
       mContainer->GetImageContainer()->CreatePlanarYCbCrImage();
-  mContainer->SetCurrentFrame(aInfo.mDisplay, blank, TimeStamp::Now());
+  mContainer->SetCurrentFrame(aInfo.mDisplay, blank, now);
+
+  if (mSecondaryContainer) {
+    mSecondaryContainer->SetCurrentFrame(aInfo.mDisplay, blank, now);
+  }
 }
 
 void VideoSink::TryUpdateRenderedVideoFrames() {
   AssertOwnerThread();
   if (mUpdateScheduler.IsScheduled() || !mAudioSink->IsPlaying()) {
     return;
   }
   RefPtr<VideoData> v = VideoQueue().PeekFront();
@@ -417,16 +429,20 @@ void VideoSink::RenderVideoFrames(int32_
 
     VSINK_LOG_V("playing video frame %" PRId64 " (id=%x) (vq-queued=%zu)",
                 frame->mTime.ToMicroseconds(), frame->mFrameID,
                 VideoQueue().GetSize());
   }
 
   if (images.Length() > 0) {
     mContainer->SetCurrentFrames(frames[0]->mDisplay, images);
+
+    if (mSecondaryContainer) {
+      mSecondaryContainer->SetCurrentFrames(frames[0]->mDisplay, images);
+    }
   }
 }
 
 void VideoSink::UpdateRenderedVideoFrames() {
   AssertOwnerThread();
   MOZ_ASSERT(mAudioSink->IsPlaying(), "should be called while playing.");
 
   // Get the current playback position.
@@ -525,16 +541,31 @@ void VideoSink::MaybeResolveEndPromise()
       } else {
         mFrameStats.NotifyPresentedFrame();
       }
     }
     mEndPromiseHolder.ResolveIfExists(true, __func__);
   }
 }
 
+void VideoSink::SetSecondaryVideoContainer(VideoFrameContainer* aSecondary) {
+  AssertOwnerThread();
+  mSecondaryContainer = aSecondary;
+  if (!IsPlaying()) {
+    // If we're paused, try to send the current frame that we're
+    // paused at to the secondary container.
+    RenderVideoFrames(1);
+  }
+}
+
+void VideoSink::ClearSecondaryVideoContainer() {
+  AssertOwnerThread();
+  mSecondaryContainer = nullptr;
+}
+
 nsCString VideoSink::GetDebugInfo() {
   AssertOwnerThread();
   auto str = nsPrintfCString(
       "VideoSink: IsStarted=%d IsPlaying=%d VideoQueue(finished=%d "
       "size=%zu) mVideoFrameEndTime=%" PRId64
       " mHasVideo=%d "
       "mVideoSinkEndRequest.Exists()=%d mEndPromiseHolder.IsEmpty()=%d",
       IsStarted(), IsPlaying(), VideoQueue().IsFinished(),
--- a/dom/media/mediasink/VideoSink.h
+++ b/dom/media/mediasink/VideoSink.h
@@ -59,16 +59,19 @@ class VideoSink : public MediaSink {
   void Stop() override;
 
   bool IsStarted() const override;
 
   bool IsPlaying() const override;
 
   void Shutdown() override;
 
+  void SetSecondaryVideoContainer(VideoFrameContainer* aSecondary) override;
+  void ClearSecondaryVideoContainer() override;
+
   nsCString GetDebugInfo() override;
 
  private:
   virtual ~VideoSink();
 
   // VideoQueue listener related.
   void OnVideoQueuePushed(RefPtr<VideoData>&& aSample);
   void OnVideoQueueFinished();
@@ -104,16 +107,17 @@ class VideoSink : public MediaSink {
   }
 
   MediaQueue<VideoData>& VideoQueue() const { return mVideoQueue; }
 
   const RefPtr<AbstractThread> mOwnerThread;
   RefPtr<MediaSink> mAudioSink;
   MediaQueue<VideoData>& mVideoQueue;
   VideoFrameContainer* mContainer;
+  RefPtr<VideoFrameContainer> mSecondaryContainer;
 
   // Producer ID to help ImageContainer distinguish different streams of
   // FrameIDs. A unique and immutable value per VideoSink.
   const ProducerID mProducerID;
 
   // Used to notify MediaDecoder's frame statistics
   FrameStatistics& mFrameStats;