Bug 1194918 - Add VideoSink which contains either AudioSinkWrapper or DecodedStreamSink as a default operating MediaSink in MDSM. r=jwwang.
authorKilik Kuo <kikuo@mozilla.com>
Mon, 19 Oct 2015 17:32:16 +0800
changeset 303539 00b1bb5ace0de6d802bd130f84dde0328b6f018c
parent 303538 ce75fb8d5a8bde7590dfdc3ee2b65b8ca6200c8e
child 303540 1dc6e120ebff05eb39ed64d6cbedbd72c0ab5bc9
push id1001
push userraliiev@mozilla.com
push dateMon, 18 Jan 2016 19:06:03 +0000
treeherdermozilla-release@8b89261f3ac4 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjwwang
bugs1194918
milestone44.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1194918 - Add VideoSink which contains either AudioSinkWrapper or DecodedStreamSink as a default operating MediaSink in MDSM. r=jwwang.
dom/media/MediaDecoderStateMachine.cpp
dom/media/MediaDecoderStateMachine.h
dom/media/mediasink/VideoSink.cpp
dom/media/mediasink/VideoSink.h
dom/media/mediasink/moz.build
--- a/dom/media/MediaDecoderStateMachine.cpp
+++ b/dom/media/MediaDecoderStateMachine.cpp
@@ -12,16 +12,17 @@
 
 #include <algorithm>
 #include <stdint.h>
 
 #include "gfx2DGlue.h"
 
 #include "mediasink/DecodedAudioDataSink.h"
 #include "mediasink/AudioSinkWrapper.h"
+#include "mediasink/VideoSink.h"
 #include "mediasink/DecodedStream.h"
 #include "mozilla/DebugOnly.h"
 #include "mozilla/Logging.h"
 #include "mozilla/mozalloc.h"
 #include "mozilla/MathAlgorithms.h"
 #include "mozilla/Preferences.h"
 #include "mozilla/SharedThreadPool.h"
 #include "mozilla/TaskQueue.h"
@@ -300,17 +301,17 @@ MediaDecoderStateMachine::MediaDecoderSt
 
   mAudioQueueListener = AudioQueue().PopEvent().Connect(
     mTaskQueue, this, &MediaDecoderStateMachine::OnAudioPopped);
   mVideoQueueListener = VideoQueue().PopEvent().Connect(
     mTaskQueue, this, &MediaDecoderStateMachine::OnVideoPopped);
 
   mMetadataManager.Connect(mReader->TimedMetadataEvent(), OwnerThread());
 
-  mMediaSink = CreateAudioSink();
+  mMediaSink = CreateMediaSink(mAudioCaptured);
 
 #ifdef MOZ_EME
   mCDMProxyPromise.Begin(mDecoder->RequestCDMProxy()->Then(
     OwnerThread(), __func__, this,
     &MediaDecoderStateMachine::OnCDMProxyReady,
     &MediaDecoderStateMachine::OnCDMProxyNotReady));
 #endif
 }
@@ -376,16 +377,33 @@ MediaDecoderStateMachine::CreateAudioSin
     MOZ_ASSERT(self->OnTaskQueue());
     return new DecodedAudioDataSink(
       self->mAudioQueue, self->GetMediaTime(),
       self->mInfo.mAudio, self->mDecoder->GetAudioChannel());
   };
   return new AudioSinkWrapper(mTaskQueue, audioSinkCreator);
 }
 
+already_AddRefed<media::MediaSink>
+MediaDecoderStateMachine::CreateMediaSink(bool aAudioCaptured)
+{
+  // TODO: We can't really create a new DecodedStream until OutputStreamManager
+  //       is extracted. It is tricky that the implementation of DecodedStream
+  //       happens to allow reuse after shutdown without creating a new one.
+  RefPtr<media::MediaSink> audioSink = aAudioCaptured ?
+    mStreamSink : CreateAudioSink();
+
+  RefPtr<media::MediaSink> mediaSink = new VideoSink(mTaskQueue,
+                                                       audioSink,
+                                                       mVideoQueue,
+                                                       mDecoder->GetVideoFrameContainer(),
+                                                       mRealTime);
+  return mediaSink.forget();
+}
+
 bool MediaDecoderStateMachine::HasFutureAudio()
 {
   MOZ_ASSERT(OnTaskQueue());
   NS_ASSERTION(HasAudio(), "Should only call HasFutureAudio() when we have audio");
   // We've got audio ready to play if:
   // 1. We've not completed playback of audio, and
   // 2. we either have more than the threshold of decoded audio available, or
   //    we've completely decoded all audio (but not finished playing it yet
@@ -1484,17 +1502,18 @@ MediaDecoderStateMachine::InvokeSeek(See
 }
 
 void MediaDecoderStateMachine::StopMediaSink()
 {
   MOZ_ASSERT(OnTaskQueue());
   if (mMediaSink->IsStarted()) {
     DECODER_LOG("Stop MediaSink");
     mMediaSink->Stop();
-    mMediaSinkPromise.DisconnectIfExists();
+    mMediaSinkAudioPromise.DisconnectIfExists();
+    mMediaSinkVideoPromise.DisconnectIfExists();
   }
 }
 
 void
 MediaDecoderStateMachine::DispatchDecodeTasksIfNeeded()
 {
   MOZ_ASSERT(OnTaskQueue());
 
@@ -1759,22 +1778,30 @@ MediaDecoderStateMachine::RequestVideoDa
 void
 MediaDecoderStateMachine::StartMediaSink()
 {
   MOZ_ASSERT(OnTaskQueue());
   if (!mMediaSink->IsStarted()) {
     mAudioCompleted = false;
     mMediaSink->Start(GetMediaTime(), mInfo);
 
-    auto promise = mMediaSink->OnEnded(TrackInfo::kAudioTrack);
-    if (promise) {
-      mMediaSinkPromise.Begin(promise->Then(
+    auto videoPromise = mMediaSink->OnEnded(TrackInfo::kVideoTrack);
+    auto audioPromise = mMediaSink->OnEnded(TrackInfo::kAudioTrack);
+
+    if (audioPromise) {
+      mMediaSinkAudioPromise.Begin(audioPromise->Then(
         OwnerThread(), __func__, this,
-        &MediaDecoderStateMachine::OnMediaSinkComplete,
-        &MediaDecoderStateMachine::OnMediaSinkError));
+        &MediaDecoderStateMachine::OnMediaSinkAudioComplete,
+        &MediaDecoderStateMachine::OnMediaSinkAudioError));
+    }
+    if (videoPromise) {
+      mMediaSinkVideoPromise.Begin(videoPromise->Then(
+        OwnerThread(), __func__, this,
+        &MediaDecoderStateMachine::OnMediaSinkVideoComplete,
+        &MediaDecoderStateMachine::OnMediaSinkVideoError));
     }
   }
 }
 
 int64_t MediaDecoderStateMachine::AudioDecodedUsecs()
 {
   MOZ_ASSERT(OnTaskQueue());
   NS_ASSERTION(HasAudio(),
@@ -2902,32 +2929,53 @@ MediaDecoderStateMachine::AudioEndTime()
   MOZ_ASSERT(OnTaskQueue());
   if (mMediaSink->IsStarted()) {
     return mMediaSink->GetEndTime(TrackInfo::kAudioTrack);
   }
   MOZ_ASSERT(!HasAudio());
   return -1;
 }
 
-void MediaDecoderStateMachine::OnMediaSinkComplete()
+void
+MediaDecoderStateMachine::OnMediaSinkVideoComplete()
+{
+  MOZ_ASSERT(OnTaskQueue());
+
+  mMediaSinkVideoPromise.Complete();
+  ScheduleStateMachine();
+}
+
+void
+MediaDecoderStateMachine::OnMediaSinkVideoError()
 {
   MOZ_ASSERT(OnTaskQueue());
 
-  mMediaSinkPromise.Complete();
+  mMediaSinkVideoPromise.Complete();
+  if (HasAudio()) {
+    return;
+  }
+  DecodeError();
+}
+
+void MediaDecoderStateMachine::OnMediaSinkAudioComplete()
+{
+  MOZ_ASSERT(OnTaskQueue());
+
+  mMediaSinkAudioPromise.Complete();
   // Set true only when we have audio.
   mAudioCompleted = mInfo.HasAudio();
   // To notify PlaybackEnded as soon as possible.
   ScheduleStateMachine();
 }
 
-void MediaDecoderStateMachine::OnMediaSinkError()
+void MediaDecoderStateMachine::OnMediaSinkAudioError()
 {
   MOZ_ASSERT(OnTaskQueue());
 
-  mMediaSinkPromise.Complete();
+  mMediaSinkAudioPromise.Complete();
   // Set true only when we have audio.
   mAudioCompleted = mInfo.HasAudio();
 
   // Make the best effort to continue playback when there is video.
   if (HasVideo()) {
     return;
   }
 
@@ -2969,20 +3017,17 @@ MediaDecoderStateMachine::SetAudioCaptur
   // Backup current playback parameters.
   MediaSink::PlaybackParams params = mMediaSink->GetPlaybackParams();
 
   // Stop and shut down the existing sink.
   StopMediaSink();
   mMediaSink->Shutdown();
 
   // Create a new sink according to whether audio is captured.
-  // TODO: We can't really create a new DecodedStream until OutputStreamManager
-  //       is extracted. It is tricky that the implementation of DecodedStream
-  //       happens to allow reuse after shutdown without creating a new one.
-  mMediaSink = aCaptured ? mStreamSink : CreateAudioSink();
+  mMediaSink = CreateMediaSink(aCaptured);
 
   // Restore playback parameters.
   mMediaSink->SetPlaybackParams(params);
 
   // We don't need to call StartMediaSink() here because IsPlaying() is now
   // always in sync with the playing state of MediaSink. It will be started in
   // MaybeStartPlayback() in the next cycle if necessary.
 
--- a/dom/media/MediaDecoderStateMachine.h
+++ b/dom/media/MediaDecoderStateMachine.h
@@ -479,16 +479,19 @@ protected:
   // arrived, otherwise sleep until it's time for the next frame. Update the
   // current frame time as appropriate, and trigger ready state update.  The
   // decoder monitor must be held with exactly one lock count. Called on the
   // state machine thread.
   void UpdateRenderedVideoFrames();
 
   media::MediaSink* CreateAudioSink();
 
+  // Always create mediasink which contains an AudioSink or StreamSink inside.
+  already_AddRefed<media::MediaSink> CreateMediaSink(bool aAudioCaptured);
+
   // Stops the media sink and shut it down.
   // The decoder monitor must be held with exactly one lock count.
   // Called on the state machine thread.
   void StopMediaSink();
 
   // Create and start the media sink.
   // The decoder monitor must be held with exactly one lock count.
   // Called on the state machine thread.
@@ -633,22 +636,24 @@ protected:
   bool IsPausedAndDecoderWaiting();
 
   // These return true if the respective stream's decode has not yet reached
   // the end of stream.
   bool IsAudioDecoding();
   bool IsVideoDecoding();
 
 private:
-  // Resolved by the MediaSink to signal that all outstanding work is complete
-  // and the sink is shutting down.
-  void OnMediaSinkComplete();
+  // Resolved by the MediaSink to signal that all audio/video outstanding
+  // work is complete and identify which part(a/v) of the sink is shutting down.
+  void OnMediaSinkAudioComplete();
+  void OnMediaSinkVideoComplete();
 
-  // Rejected by the MediaSink to signal errors.
-  void OnMediaSinkError();
+  // Rejected by the MediaSink to signal errors for audio/video.
+  void OnMediaSinkAudioError();
+  void OnMediaSinkVideoError();
 
   // Return true if the video decoder's decode speed can not catch up the
   // play time.
   bool NeedToSkipToNextKeyframe();
 
   void AdjustAudioThresholds();
 
   // The decoder object that created this state machine. The state machine
@@ -1187,17 +1192,19 @@ private:
   // Only written on the main thread while holding the monitor. Therefore it
   // can be read on any thread while holding the monitor, or on the main thread
   // without holding the monitor.
   RefPtr<DecodedStream> mStreamSink;
 
   // Media data resource from the decoder.
   RefPtr<MediaResource> mResource;
 
-  MozPromiseRequestHolder<GenericPromise> mMediaSinkPromise;
+  // Track the complete & error for audio/video separately
+  MozPromiseRequestHolder<GenericPromise> mMediaSinkAudioPromise;
+  MozPromiseRequestHolder<GenericPromise> mMediaSinkVideoPromise;
 
   MediaEventListener mAudioQueueListener;
   MediaEventListener mVideoQueueListener;
 
   // True if audio is offloading.
   // Playback will not start when audio is offloading.
   bool mAudioOffloading;
 
new file mode 100644
--- /dev/null
+++ b/dom/media/mediasink/VideoSink.cpp
@@ -0,0 +1,182 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "VideoSink.h"
+
+namespace mozilla {
+
+extern PRLogModuleInfo* gMediaDecoderLog;
+#define VSINK_LOG(msg, ...) \
+  MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, \
+    ("VideoSink=%p " msg, this, ##__VA_ARGS__))
+#define VSINK_LOG_V(msg, ...) \
+  MOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, \
+  ("VideoSink=%p " msg, this, ##__VA_ARGS__))
+
+using namespace mozilla::layers;
+
+namespace media {
+
+VideoSink::~VideoSink()
+{
+}
+
+const MediaSink::PlaybackParams&
+VideoSink::GetPlaybackParams() const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+  return mAudioSink->GetPlaybackParams();
+}
+
+void
+VideoSink::SetPlaybackParams(const PlaybackParams& aParams)
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+  mAudioSink->SetPlaybackParams(aParams);
+}
+
+RefPtr<GenericPromise>
+VideoSink::OnEnded(TrackType aType)
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+  MOZ_ASSERT(mAudioSink->IsStarted(), "Must be called after playback starts.");
+
+  if (aType == TrackInfo::kAudioTrack) {
+    return mAudioSink->OnEnded(aType);
+  } else if (aType == TrackInfo::kVideoTrack) {
+    return mEndPromise;
+  }
+  return nullptr;
+}
+
+int64_t
+VideoSink::GetEndTime(TrackType aType) const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+  MOZ_ASSERT(mAudioSink->IsStarted(), "Must be called after playback starts.");
+
+  if (aType == TrackInfo::kVideoTrack) {
+    return mVideoFrameEndTime;
+  } else if (aType == TrackInfo::kAudioTrack) {
+    return mAudioSink->GetEndTime(aType);
+  }
+  return -1;
+}
+
+int64_t
+VideoSink::GetPosition(TimeStamp* aTimeStamp) const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+
+  return mAudioSink->GetPosition(aTimeStamp);
+}
+
+bool
+VideoSink::HasUnplayedFrames(TrackType aType) const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+  MOZ_ASSERT(aType == TrackInfo::kAudioTrack, "Not implemented for non audio tracks.");
+
+  return mAudioSink->HasUnplayedFrames(aType);
+}
+
+void
+VideoSink::SetPlaybackRate(double aPlaybackRate)
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+
+  mAudioSink->SetPlaybackRate(aPlaybackRate);
+}
+
+void
+VideoSink::SetPlaying(bool aPlaying)
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+
+  VSINK_LOG_V(" playing (%d) -> (%d)", mAudioSink->IsPlaying(), aPlaying);
+
+  mAudioSink->SetPlaying(aPlaying);
+}
+
+void
+VideoSink::Start(int64_t aStartTime, const MediaInfo& aInfo)
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+  VSINK_LOG("[%s]", __func__);
+  mAudioSink->Start(aStartTime, aInfo);
+
+  if (aInfo.HasVideo()) {
+    mEndPromise = mEndPromiseHolder.Ensure(__func__);
+    mVideoSinkEndRequest.Begin(mEndPromise->Then(
+      mOwnerThread.get(), __func__, this,
+      &VideoSink::OnVideoEnded,
+      &VideoSink::OnVideoEnded));
+  }
+}
+
+void
+VideoSink::OnVideoEnded()
+{
+  AssertOwnerThread();
+
+  mVideoSinkEndRequest.Complete();
+}
+
+void
+VideoSink::Stop()
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+  MOZ_ASSERT(mAudioSink->IsStarted(), "playback not started.");
+  VSINK_LOG("[%s]", __func__);
+
+  mAudioSink->Stop();
+
+  mVideoSinkEndRequest.DisconnectIfExists();
+  mEndPromiseHolder.ResolveIfExists(true, __func__);
+  mEndPromise = nullptr;
+}
+
+bool
+VideoSink::IsStarted() const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+
+  return mAudioSink->IsStarted();
+}
+
+bool
+VideoSink::IsPlaying() const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+
+  return mAudioSink->IsPlaying();
+}
+
+void
+VideoSink::Shutdown()
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(mAudioSink, "AudioSink should exist.");
+  MOZ_ASSERT(!mAudioSink->IsStarted(), "must be called after playback stops.");
+  VSINK_LOG("[%s]", __func__);
+
+  mAudioSink->Shutdown();
+}
+
+} // namespace media
+} // namespace mozilla
new file mode 100644
--- /dev/null
+++ b/dom/media/mediasink/VideoSink.h
@@ -0,0 +1,101 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef VideoSink_h_
+#define VideoSink_h_
+
+#include "ImageContainer.h"
+#include "MediaSink.h"
+#include "mozilla/AbstractThread.h"
+#include "mozilla/MozPromise.h"
+#include "mozilla/RefPtr.h"
+#include "mozilla/TimeStamp.h"
+#include "VideoFrameContainer.h"
+
+namespace mozilla {
+
+class VideoFrameContainer;
+template <class T> class MediaQueue;
+
+namespace media {
+
+class VideoSink : public MediaSink
+{
+public:
+  VideoSink(AbstractThread* aThread,
+            MediaSink* aAudioSink,
+            MediaQueue<MediaData>& aVideoQueue,
+            VideoFrameContainer* aContainer,
+            bool aRealTime)
+    : mOwnerThread(aThread)
+    , mAudioSink(aAudioSink)
+    , mVideoQueue(aVideoQueue)
+    , mContainer(aContainer)
+    , mRealTime(aRealTime)
+    , mVideoFrameEndTime(-1)
+  {}
+
+  const PlaybackParams& GetPlaybackParams() const override;
+
+  void SetPlaybackParams(const PlaybackParams& aParams) override;
+
+  RefPtr<GenericPromise> OnEnded(TrackType aType) override;
+
+  int64_t GetEndTime(TrackType aType) const override;
+
+  int64_t GetPosition(TimeStamp* aTimeStamp = nullptr) const override;
+
+  bool HasUnplayedFrames(TrackType aType) const override;
+
+  void SetPlaybackRate(double aPlaybackRate) override;
+
+  void SetPlaying(bool aPlaying) override;
+
+  void Start(int64_t aStartTime, const MediaInfo& aInfo) override;
+
+  void Stop() override;
+
+  bool IsStarted() const override;
+
+  bool IsPlaying() const override;
+
+  void Shutdown() override;
+
+private:
+  virtual ~VideoSink();
+
+  void AssertOwnerThread() const
+  {
+    MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
+  }
+
+  MediaQueue<MediaData>& VideoQueue() const {
+    return mVideoQueue;
+  }
+
+  void OnVideoEnded();
+
+  const RefPtr<AbstractThread> mOwnerThread;
+  RefPtr<MediaSink> mAudioSink;
+  MediaQueue<MediaData>& mVideoQueue;
+  VideoFrameContainer* mContainer;
+
+  // True if we are decoding a real-time stream.
+  const bool mRealTime;
+
+  RefPtr<GenericPromise> mEndPromise;
+  MozPromiseHolder<GenericPromise> mEndPromiseHolder;
+  MozPromiseRequestHolder<GenericPromise> mVideoSinkEndRequest;
+
+  // The presentation end time of the last video frame which has been displayed
+  // in microseconds.
+  int64_t mVideoFrameEndTime;
+};
+
+} // namespace media
+} // namespace mozilla
+
+#endif
--- a/dom/media/mediasink/moz.build
+++ b/dom/media/mediasink/moz.build
@@ -3,11 +3,12 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 UNIFIED_SOURCES += [
     'AudioSinkWrapper.cpp',
     'DecodedAudioDataSink.cpp',
     'DecodedStream.cpp',
+    'VideoSink.cpp',
 ]
 
 FINAL_LIBRARY = 'xul'