Bug 1199121. Part 1 - add the ability to handle video-only streams to AudioSinkWrapper. r=kinetik.
authorJW Wang <jwwang@mozilla.com>
Mon, 07 Sep 2015 11:56:52 +0800
changeset 290891 d71e93b86d1fc8e4ed23b2de65b7a3c5af1bec6f
parent 290890 365a1e874a93976b5af8109e33cd6780c794e6f2
child 290892 11b926fd7b6b50df2b83adee427a86471c8c71b6
push id5188
push userdburns@mozilla.com
push dateTue, 08 Sep 2015 12:16:07 +0000
reviewerskinetik
bugs1199121
milestone43.0a1
Bug 1199121. Part 1 - add the ability to handle video-only streams to AudioSinkWrapper. r=kinetik. Note AudioSinkWrapper can now report correct playback position when the media is video-only or audio-only. We will handle the case where audio ends before video in next patch where it needs to switch to system clock when audio reaches the end.
dom/media/MediaDecoderStateMachine.cpp
dom/media/mediasink/AudioSinkWrapper.cpp
dom/media/mediasink/AudioSinkWrapper.h
dom/media/mediasink/MediaSink.h
--- a/dom/media/MediaDecoderStateMachine.cpp
+++ b/dom/media/MediaDecoderStateMachine.cpp
@@ -1756,25 +1756,27 @@ MediaDecoderStateMachine::StartAudioSink
 {
   MOZ_ASSERT(OnTaskQueue());
   AssertCurrentThreadInMonitor();
   if (mAudioCaptured) {
     MOZ_ASSERT(!mAudioSink->IsStarted());
     return;
   }
 
-  if (HasAudio() && !mAudioSink->IsStarted()) {
+  if (!mAudioSink->IsStarted()) {
     mAudioCompleted = false;
     mAudioSink->Start(GetMediaTime(), mInfo);
 
-    mAudioSinkPromise.Begin(
-      mAudioSink->OnEnded(TrackInfo::kAudioTrack)->Then(
+    auto promise = mAudioSink->OnEnded(TrackInfo::kAudioTrack);
+    if (promise) {
+      mAudioSinkPromise.Begin(promise->Then(
         OwnerThread(), __func__, this,
         &MediaDecoderStateMachine::OnAudioSinkComplete,
         &MediaDecoderStateMachine::OnAudioSinkError));
+    }
   }
 }
 
 void
 MediaDecoderStateMachine::StopDecodedStream()
 {
   MOZ_ASSERT(OnTaskQueue());
   AssertCurrentThreadInMonitor();
@@ -2615,22 +2617,18 @@ int64_t MediaDecoderStateMachine::GetClo
   // fed to a MediaStream, use that stream as the source of the clock.
   int64_t clock_time = -1;
   TimeStamp t;
   if (!IsPlaying()) {
     clock_time = mPlayDuration;
   } else {
     if (mAudioCaptured) {
       clock_time = GetStreamClock();
-    } else if (HasAudio() && !mAudioCompleted) {
-      clock_time = GetAudioClock();
     } else {
-      t = TimeStamp::Now();
-      // Audio is disabled on this system. Sync to the system clock.
-      clock_time = GetVideoStreamPosition(t);
+      clock_time = mAudioSink->GetPosition(&t);
     }
     NS_ASSERTION(GetMediaTime() <= clock_time, "Clock should go forwards.");
   }
   if (aTimeStamp) {
     *aTimeStamp = t.IsNull() ? TimeStamp::Now() : t;
   }
 
   return clock_time;
@@ -2993,27 +2991,16 @@ MediaDecoderStateMachine::LogicalPlaybac
   MOZ_ASSERT(OnTaskQueue());
   ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
 
   if (mLogicalPlaybackRate == 0) {
     // This case is handled in MediaDecoder by pausing playback.
     return;
   }
 
-  // AudioStream will handle playback rate change when we have audio.
-  // Do nothing while we are not playing. Change in playback rate will
-  // take effect next time we start playing again.
-  if (!HasAudio() && IsPlaying()) {
-    // Remember how much time we've spent in playing the media
-    // for playback rate will change from now on.
-    TimeStamp now = TimeStamp::Now();
-    mPlayDuration = GetVideoStreamPosition(now);
-    SetPlayStartTime(now);
-  }
-
   mPlaybackRate = mLogicalPlaybackRate;
   mAudioSink->SetPlaybackRate(mPlaybackRate);
 
   ScheduleStateMachine();
 }
 
 void MediaDecoderStateMachine::PreservesPitchChanged()
 {
@@ -3044,28 +3031,26 @@ MediaDecoderStateMachine::AudioEndTime()
 
 void MediaDecoderStateMachine::OnAudioSinkComplete()
 {
   MOZ_ASSERT(OnTaskQueue());
   ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
   MOZ_ASSERT(!mAudioCaptured, "Should be disconnected when capturing audio.");
 
   mAudioSinkPromise.Complete();
-  ResyncAudioClock();
   mAudioCompleted = true;
 }
 
 void MediaDecoderStateMachine::OnAudioSinkError()
 {
   MOZ_ASSERT(OnTaskQueue());
   ReentrantMonitorAutoEnter mon(mDecoder->GetReentrantMonitor());
   MOZ_ASSERT(!mAudioCaptured, "Should be disconnected when capturing audio.");
 
   mAudioSinkPromise.Complete();
-  ResyncAudioClock();
   mAudioCompleted = true;
 
   // Make the best effort to continue playback when there is video.
   if (HasVideo()) {
     return;
   }
 
   // Otherwise notify media decoder/element about this error for it makes
--- a/dom/media/mediasink/AudioSinkWrapper.cpp
+++ b/dom/media/mediasink/AudioSinkWrapper.cpp
@@ -59,21 +59,51 @@ AudioSinkWrapper::GetEndTime(TrackType a
   MOZ_ASSERT(mIsStarted, "Must be called after playback starts.");
   if (aType == TrackInfo::kAudioTrack && mAudioSink) {
     return mAudioSink->GetEndTime();
   }
   return -1;
 }
 
 int64_t
-AudioSinkWrapper::GetPosition() const
+AudioSinkWrapper::GetVideoPosition(TimeStamp aNow) const
+{
+  AssertOwnerThread();
+  MOZ_ASSERT(!mPlayStartTime.IsNull());
+  // Time elapsed since we started playing.
+  int64_t delta = (aNow - mPlayStartTime).ToMicroseconds();
+  // Take playback rate into account.
+  return mPlayDuration + delta * mParams.playbackRate;
+}
+
+int64_t
+AudioSinkWrapper::GetPosition(TimeStamp* aTimeStamp) const
 {
   AssertOwnerThread();
   MOZ_ASSERT(mIsStarted, "Must be called after playback starts.");
-  return mAudioSink->GetPosition();
+
+  int64_t pos = -1;
+  TimeStamp t = TimeStamp::Now();
+
+  if (mAudioSink) {
+    // Rely on the audio sink to report playback position when we have one.
+    pos = mAudioSink->GetPosition();
+  } else if (!mPlayStartTime.IsNull()) {
+    // Calculate playback position using system clock if we are still playing.
+    pos = GetVideoPosition(t);
+  } else {
+    // Return how long we've played if we are not playing.
+    pos = mPlayDuration;
+  }
+
+  if (aTimeStamp) {
+    *aTimeStamp = t;
+  }
+
+  return pos;
 }
 
 bool
 AudioSinkWrapper::HasUnplayedFrames(TrackType aType) const
 {
   AssertOwnerThread();
   return mAudioSink ? mAudioSink->HasUnplayedFrames() : false;
 }
@@ -89,18 +119,27 @@ AudioSinkWrapper::SetVolume(double aVolu
 }
 
 void
 AudioSinkWrapper::SetPlaybackRate(double aPlaybackRate)
 {
   AssertOwnerThread();
   mParams.playbackRate = aPlaybackRate;
   if (mAudioSink) {
+    // Pass the playback rate to the audio sink. The underlying AudioStream
+    // will handle playback rate changes and report correct audio position.
     mAudioSink->SetPlaybackRate(aPlaybackRate);
+  } else if (!mPlayStartTime.IsNull()) {
+    // Adjust playback duration and start time when we are still playing.
+    TimeStamp now = TimeStamp::Now();
+    mPlayDuration = GetVideoPosition(now);
+    mPlayStartTime = now;
   }
+  // Do nothing when not playing. Changes in playback rate will be taken into
+  // account by GetVideoPosition().
 }
 
 void
 AudioSinkWrapper::SetPreservesPitch(bool aPreservesPitch)
 {
   AssertOwnerThread();
   mParams.preservesPitch = aPreservesPitch;
   if (mAudioSink) {
@@ -116,41 +155,59 @@ AudioSinkWrapper::SetPlaying(bool aPlayi
   // Resume/pause matters only when playback started.
   if (!mIsStarted) {
     return;
   }
 
   if (mAudioSink) {
     mAudioSink->SetPlaying(aPlaying);
   }
+
+  if (aPlaying) {
+    MOZ_ASSERT(mPlayStartTime.IsNull());
+    mPlayStartTime = TimeStamp::Now();
+  } else {
+    // Remember how long we've played.
+    mPlayDuration = GetPosition();
+    // mPlayStartTime must be updated later since GetPosition()
+    // depends on the value of mPlayStartTime.
+    mPlayStartTime = TimeStamp();
+  }
 }
 
 void
 AudioSinkWrapper::Start(int64_t aStartTime, const MediaInfo& aInfo)
 {
   AssertOwnerThread();
   MOZ_ASSERT(!mIsStarted, "playback already started.");
 
   mIsStarted = true;
+  mPlayDuration = aStartTime;
+  mPlayStartTime = TimeStamp::Now();
 
-  mAudioSink = mCreator->Create();
-  mEndPromise = mAudioSink->Init();
-  SetPlaybackParams(mParams);
+  if (aInfo.HasAudio()) {
+    mAudioSink = mCreator->Create();
+    mEndPromise = mAudioSink->Init();
+    SetPlaybackParams(mParams);
+  }
 }
 
 void
 AudioSinkWrapper::Stop()
 {
   AssertOwnerThread();
   MOZ_ASSERT(mIsStarted, "playback not started.");
 
   mIsStarted = false;
-  mAudioSink->Shutdown();
-  mAudioSink = nullptr;
-  mEndPromise = nullptr;
+
+  if (mAudioSink) {
+    mAudioSink->Shutdown();
+    mAudioSink = nullptr;
+    mEndPromise = nullptr;
+  }
 }
 
 bool
 AudioSinkWrapper::IsStarted() const
 {
   AssertOwnerThread();
   return mIsStarted;
 }
--- a/dom/media/mediasink/AudioSinkWrapper.h
+++ b/dom/media/mediasink/AudioSinkWrapper.h
@@ -5,16 +5,17 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef AudioSinkWrapper_h_
 #define AudioSinkWrapper_h_
 
 #include "mozilla/AbstractThread.h"
 #include "mozilla/dom/AudioChannelBinding.h"
 #include "mozilla/nsRefPtr.h"
+#include "mozilla/TimeStamp.h"
 #include "mozilla/UniquePtr.h"
 
 #include "MediaSink.h"
 
 namespace mozilla {
 
 class MediaData;
 template <class T> class MediaQueue;
@@ -45,24 +46,26 @@ class AudioSinkWrapper : public MediaSin
   };
 
 public:
   template <typename Function>
   AudioSinkWrapper(AbstractThread* aOwnerThread, const Function& aFunc)
     : mOwnerThread(aOwnerThread)
     , mCreator(new CreatorImpl<Function>(aFunc))
     , mIsStarted(false)
+    // Give an insane value to facilitate debug if used before playback starts.
+    , mPlayDuration(INT64_MAX)
   {}
 
   const PlaybackParams& GetPlaybackParams() const override;
   void SetPlaybackParams(const PlaybackParams& aParams) override;
 
   nsRefPtr<GenericPromise> OnEnded(TrackType aType) override;
   int64_t GetEndTime(TrackType aType) const override;
-  int64_t GetPosition() const override;
+  int64_t GetPosition(TimeStamp* aTimeStamp = nullptr) const override;
   bool HasUnplayedFrames(TrackType aType) const override;
 
   void SetVolume(double aVolume) override;
   void SetPlaybackRate(double aPlaybackRate) override;
   void SetPreservesPitch(bool aPreservesPitch) override;
   void SetPlaying(bool aPlaying) override;
 
   void Start(int64_t aStartTime, const MediaInfo& aInfo) override;
@@ -73,21 +76,26 @@ public:
 
 private:
   virtual ~AudioSinkWrapper();
 
   void AssertOwnerThread() const {
     MOZ_ASSERT(mOwnerThread->IsCurrentThreadIn());
   }
 
+  int64_t GetVideoPosition(TimeStamp aNow) const;
+
   const nsRefPtr<AbstractThread> mOwnerThread;
   UniquePtr<Creator> mCreator;
   nsRefPtr<AudioSink> mAudioSink;
   nsRefPtr<GenericPromise> mEndPromise;
 
   bool mIsStarted;
   PlaybackParams mParams;
+
+  TimeStamp mPlayStartTime;
+  int64_t mPlayDuration;
 };
 
 } // namespace media
 } // namespace mozilla
 
 #endif //AudioSinkWrapper_h_
--- a/dom/media/mediasink/MediaSink.h
+++ b/dom/media/mediasink/MediaSink.h
@@ -8,16 +8,19 @@
 #define MediaSink_h_
 
 #include "mozilla/nsRefPtr.h"
 #include "mozilla/MozPromise.h"
 #include "nsISupportsImpl.h"
 #include "MediaInfo.h"
 
 namespace mozilla {
+
+class TimeStamp;
+
 namespace media {
 
 /**
  * A consumer of audio/video data which plays audio and video tracks and
  * manages A/V sync between them.
  *
  * A typical sink sends audio/video outputs to the speaker and screen.
  * However, there are also sinks which capture the output of an media element
@@ -59,18 +62,20 @@ public:
   // Return the end time of the audio/video data that has been consumed
   // or -1 if no such track.
   // Must be called after playback starts.
   virtual int64_t GetEndTime(TrackType aType) const = 0;
 
   // Return playback position of the media.
   // Since A/V sync is always maintained by this sink, there is no need to
   // specify whether we want to get audio or video position.
+  // aTimeStamp returns the timeStamp corresponding to the returned position
+  // which is used by the compositor to derive the render time of video frames.
   // Must be called after playback starts.
-  virtual int64_t GetPosition() const = 0;
+  virtual int64_t GetPosition(TimeStamp* aTimeStamp = nullptr) const = 0;
 
   // Return true if there are data consumed but not played yet.
   // Can be called in any state.
   virtual bool HasUnplayedFrames(TrackType aType) const = 0;
 
   // Set volume of the audio track.
   // Do nothing if this sink has no audio track.
   // Can be called in any state.