Bug 1509548 - Make MediaStreamGraph pull data per track instead of per stream. r=padenot
authorAndreas Pehrson <apehrson@mozilla.com>
Thu, 29 Nov 2018 17:37:42 +0000
changeset 505196 daaecb62f373d33c60e6a77bcce01d27e19a3282
parent 505195 311cee86bc66141b8f59be67cd203018f5799cae
child 505197 daf1ffb72989d83be571e2bfb614dbee639915ce
push id10290
push userffxbld-merge
push dateMon, 03 Dec 2018 16:23:23 +0000
treeherdermozilla-beta@700bed2445e6 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerspadenot
bugs1509548
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1509548 - Make MediaStreamGraph pull data per track instead of per stream. r=padenot Differential Revision: https://phabricator.services.mozilla.com/D13086
dom/media/CanvasCaptureMediaStream.cpp
dom/media/MediaManager.cpp
dom/media/MediaStreamGraph.cpp
dom/media/MediaStreamGraph.h
media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
--- a/dom/media/CanvasCaptureMediaStream.cpp
+++ b/dom/media/CanvasCaptureMediaStream.cpp
@@ -110,17 +110,17 @@ OutputStreamDriver::OutputStreamDriver(S
     : FrameCaptureListener(),
       mSourceStream(aSourceStream),
       mTrackListener(
           new TrackListener(aTrackId, aPrincipalHandle, aSourceStream)) {
   MOZ_ASSERT(NS_IsMainThread());
   MOZ_ASSERT(mSourceStream);
   mSourceStream->AddTrack(aTrackId, new VideoSegment());
   mSourceStream->AddTrackListener(mTrackListener, aTrackId);
-  mSourceStream->SetPullEnabled(true);
+  mSourceStream->SetPullingEnabled(aTrackId, true);
 
   // All CanvasCaptureMediaStreams shall at least get one frame.
   mFrameCaptureRequested = true;
 }
 
 OutputStreamDriver::~OutputStreamDriver() {
   MOZ_ASSERT(NS_IsMainThread());
   // MediaStreamGraph will keep the listener alive until it can end the track in
--- a/dom/media/MediaManager.cpp
+++ b/dom/media/MediaManager.cpp
@@ -4202,30 +4202,37 @@ RefPtr<SourceListener::InitPromise> Sour
   return init->Then(
       GetMainThreadSerialEventTarget(), __func__,
       [self = RefPtr<SourceListener>(this), this]() {
         if (mStopped) {
           // We were shut down during the async init
           return InitPromise::CreateAndResolve(true, __func__);
         }
 
-        mStream->SetPullEnabled(true);
-
         for (DeviceState* state :
              {mAudioDeviceState.get(), mVideoDeviceState.get()}) {
           if (!state) {
             continue;
           }
           MOZ_DIAGNOSTIC_ASSERT(!state->mTrackEnabled);
           MOZ_DIAGNOSTIC_ASSERT(!state->mDeviceEnabled);
           MOZ_DIAGNOSTIC_ASSERT(!state->mStopped);
 
           state->mDeviceEnabled = true;
           state->mTrackEnabled = true;
           state->mTrackEnabledTime = TimeStamp::Now();
+
+          if (state->mDevice->GetMediaSource() !=
+              MediaSourceEnum::AudioCapture) {
+            // For AudioCapture mStream is a dummy stream, so we don't try to
+            // enable pulling - there won't be a track to enable it for.
+            mStream->SetPullingEnabled(
+                state == mAudioDeviceState.get() ? kAudioTrack : kVideoTrack,
+                true);
+          }
         }
         return InitPromise::CreateAndResolve(true, __func__);
       },
       [self = RefPtr<SourceListener>(this),
        this](RefPtr<MediaMgrError>&& aResult) {
         if (mStopped) {
           return InitPromise::CreateAndReject(std::move(aResult), __func__);
         }
@@ -4290,21 +4297,22 @@ void SourceListener::Remove() {
   mWindowListener = nullptr;
 
   // If it's destroyed, don't call - listener will be removed and we'll be
   // notified!
   if (!mStream->IsDestroyed()) {
     // We disable pulling before removing so we don't risk having live tracks
     // without a listener attached - that wouldn't produce data and would be
     // illegal to the graph.
-    mStream->SetPullEnabled(false);
     if (mAudioDeviceState) {
+      mStream->SetPullingEnabled(kAudioTrack, false);
       mStream->RemoveTrackListener(mAudioDeviceState->mListener, kAudioTrack);
     }
     if (mVideoDeviceState) {
+      mStream->SetPullingEnabled(kVideoTrack, false);
       mStream->RemoveTrackListener(mVideoDeviceState->mListener, kVideoTrack);
     }
   }
 
   if (mAudioDeviceState) {
     mAudioDeviceState->mListener = nullptr;
   }
   if (mVideoDeviceState) {
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -1216,22 +1216,31 @@ void MediaStreamGraphImpl::UpdateGraph(G
              MediaTimeToSeconds(endTime)));
         // Data can't be added to a finished stream, so underruns are
         // irrelevant.
         stream->mStartBlocking = std::min(endTime, aEndBlockingDecisions);
       }
     } else {
       stream->mStartBlocking = WillUnderrun(stream, aEndBlockingDecisions);
 
-      SourceMediaStream* s = stream->AsSourceStream();
-      if (s && s->mPullEnabled) {
+#ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
+      if (SourceMediaStream* s = stream->AsSourceStream()) {
         for (StreamTracks::TrackIter i(s->mTracks); !i.IsEnded(); i.Next()) {
           if (i->IsEnded()) {
             continue;
           }
+          SourceMediaStream::TrackData* data;
+          {
+            MutexAutoLock lock(s->mMutex);
+            data = s->FindDataForTrack(i->GetID());
+          }
+          MOZ_ASSERT(data);
+          if (!data->mPullingEnabled) {
+            continue;
+          }
           if (i->GetEnd() <
               stream->GraphTimeToStreamTime(aEndBlockingDecisions)) {
             LOG(LogLevel::Error,
                 ("%p: SourceMediaStream %p track %u (%s) is live and pulled, "
                  "but wasn't fed "
                  "enough data. TrackListeners=%zu. Track-end=%f, "
                  "Iteration-end=%f",
                  this, stream, i->GetID(),
@@ -1241,16 +1250,17 @@ void MediaStreamGraphImpl::UpdateGraph(G
                  MediaTimeToSeconds(
                      stream->GraphTimeToStreamTime(aEndBlockingDecisions))));
             MOZ_DIAGNOSTIC_ASSERT(false,
                                   "A non-finished SourceMediaStream wasn't fed "
                                   "enough data by NotifyPull");
           }
         }
       }
+#endif /* MOZ_DIAGNOSTIC_ASSERT_ENABLED */
     }
   }
 
   for (MediaStream* stream : mSuspendedStreams) {
     stream->mStartBlocking = mStateComputedTime;
   }
 
   // If the loop is woken up so soon that IterationEnd() barely advances or
@@ -2454,17 +2464,16 @@ void MediaStream::AddMainThreadListener(
 
   nsCOMPtr<nsIRunnable> runnable = new NotifyRunnable(this);
   GraphImpl()->Dispatch(runnable.forget());
 }
 
 SourceMediaStream::SourceMediaStream()
     : MediaStream(),
       mMutex("mozilla::media::SourceMediaStream"),
-      mPullEnabled(false),
       mFinishPending(false) {}
 
 nsresult SourceMediaStream::OpenAudioInput(CubebUtils::AudioDeviceID aID,
                                            AudioDataListener* aListener) {
   MOZ_ASSERT(GraphImpl());
   mInputListener = aListener;
   return GraphImpl()->OpenAudioInput(aID, aListener);
 }
@@ -2491,46 +2500,63 @@ void SourceMediaStream::DestroyImpl() {
   }
 
   // Hold mMutex while mGraph is reset so that other threads holding mMutex
   // can null-check know that the graph will not destroyed.
   MutexAutoLock lock(mMutex);
   MediaStream::DestroyImpl();
 }
 
-void SourceMediaStream::SetPullEnabled(bool aEnabled) {
+void SourceMediaStream::SetPullingEnabled(TrackID aTrackID, bool aEnabled) {
   class Message : public ControlMessage {
    public:
-    Message(SourceMediaStream* aStream, bool aEnabled)
-        : ControlMessage(nullptr), mStream(aStream), mEnabled(aEnabled) {}
+    Message(SourceMediaStream* aStream, TrackID aTrackID, bool aEnabled)
+        : ControlMessage(nullptr),
+          mStream(aStream),
+          mTrackID(aTrackID),
+          mEnabled(aEnabled) {}
     void Run() override {
       MutexAutoLock lock(mStream->mMutex);
-      mStream->mPullEnabled = mEnabled;
+      TrackData* data = mStream->FindDataForTrack(mTrackID);
+      if (!data) {
+        // We can't enable pulling for a track that was never added. We ignore
+        // this if we're disabling pulling, since shutdown sequences are
+        // complex. If there's truly an issue we'll have issues enabling anyway.
+        MOZ_ASSERT_IF(mEnabled,
+                      mStream->mTracks.FindTrack(mTrackID) &&
+                          mStream->mTracks.FindTrack(mTrackID)->IsEnded());
+        return;
+      }
+      data->mPullingEnabled = mEnabled;
     }
     SourceMediaStream* mStream;
+    TrackID mTrackID;
     bool mEnabled;
   };
-  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aEnabled));
+  GraphImpl()->AppendMessage(MakeUnique<Message>(this, aTrackID, aEnabled));
 }
 
 bool SourceMediaStream::PullNewData(GraphTime aDesiredUpToTime) {
   TRACE_AUDIO_CALLBACK_COMMENT("SourceMediaStream %p", this);
   MutexAutoLock lock(mMutex);
-  if (!mPullEnabled || mFinished) {
+  if (mFinished) {
     return false;
   }
   // Compute how much stream time we'll need assuming we don't block
   // the stream at all.
   StreamTime t = GraphTimeToStreamTime(aDesiredUpToTime);
   StreamTime current = mTracks.GetEarliestTrackEnd();
   LOG(LogLevel::Verbose,
       ("%p: Calling NotifyPull aStream=%p t=%f current end=%f", GraphImpl(),
        this, GraphImpl()->MediaTimeToSeconds(t),
        GraphImpl()->MediaTimeToSeconds(current)));
   for (const TrackData& track : mUpdateTracks) {
+    if (!track.mPullingEnabled) {
+      continue;
+    }
     if (track.mCommands & TrackEventCommand::TRACK_EVENT_ENDED) {
       continue;
     }
     current = track.mEndOfFlushedData + track.mData->GetDuration();
     if (t <= current) {
       continue;
     }
     MutexAutoUnlock unlock(mMutex);
@@ -2618,16 +2644,17 @@ void SourceMediaStream::AddTrackInternal
       ("%p: AddTrackInternal: %lu/%lu", GraphImpl(),
        (long)mPendingTracks.Length(), (long)mUpdateTracks.Length()));
   data->mID = aID;
   data->mInputRate = aRate;
   data->mResamplerChannelCount = 0;
   data->mEndOfFlushedData = 0;
   data->mCommands = TRACK_CREATE;
   data->mData = aSegment;
+  data->mPullingEnabled = false;
   ResampleAudioToGraphSampleRate(data, aSegment);
   if (!(aFlags & ADDTRACK_QUEUED) && GraphImpl()) {
     GraphImpl()->EnsureNextIteration();
   }
 }
 
 void SourceMediaStream::AddAudioTrack(TrackID aID, TrackRate aRate,
                                       AudioSegment* aSegment, uint32_t aFlags) {
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -653,23 +653,24 @@ class SourceMediaStream : public MediaSt
  public:
   explicit SourceMediaStream();
 
   SourceMediaStream* AsSourceStream() override { return this; }
 
   // Main thread only
 
   /**
-   * Enable or disable pulling. When pulling is enabled, NotifyPull
-   * gets called on MediaStream/TrackListeners for this stream during the
-   * MediaStreamGraph control loop. Pulling is initially disabled.
-   * Due to unavoidable race conditions, after a call to SetPullEnabled(false)
+   * Enable or disable pulling for a specific track.
+   * When pulling is enabled, NotifyPull gets called on the corresponding
+   * MediaStreamTrackListeners for this stream during the MediaStreamGraph
+   * control loop. Pulling is initially disabled for all tracks. Due to
+   * unavoidable race conditions, after a call to SetPullingEnabled(false)
    * it is still possible for a NotifyPull to occur.
    */
-  void SetPullEnabled(bool aEnabled);
+  void SetPullingEnabled(TrackID aTrackID, bool aEnabled);
 
   // Users of audio inputs go through the stream so it can track when the
   // last stream referencing an input goes away, so it can close the cubeb
   // input.  Also note: callable on any thread (though it bounces through
   // MainThread to set the command if needed).
   nsresult OpenAudioInput(CubebUtils::AudioDeviceID aID,
                           AudioDataListener* aListener);
   // Note: also implied when Destroy() happens
@@ -804,16 +805,18 @@ class SourceMediaStream : public MediaSt
     // End-time of data already flushed to the track (excluding mData)
     StreamTime mEndOfFlushedData;
     // Each time the track updates are flushed to the media graph thread,
     // the segment buffer is emptied.
     nsAutoPtr<MediaSegment> mData;
     // Each time the track updates are flushed to the media graph thread,
     // this is cleared.
     uint32_t mCommands;
+    // True if the producer of this track is having data pulled by the graph.
+    bool mPullingEnabled;
   };
 
   bool NeedsMixing();
 
   void ResampleAudioToGraphSampleRate(TrackData* aTrackData,
                                       MediaSegment* aSegment);
 
   void AddDirectTrackListenerImpl(
@@ -862,17 +865,16 @@ class SourceMediaStream : public MediaSt
   // protected by mMutex
   // This time stamp will be updated in adding and blocked SourceMediaStream,
   // |AddStreamGraphThread| and |AdvanceTimeVaryingValuesToCurrentTime| in
   // particularly.
   TimeStamp mStreamTracksStartTimeStamp;
   nsTArray<TrackData> mUpdateTracks;
   nsTArray<TrackData> mPendingTracks;
   nsTArray<TrackBound<DirectMediaStreamTrackListener>> mDirectTrackListeners;
-  bool mPullEnabled;
   bool mFinishPending;
 };
 
 /**
  * The blocking mode decides how a track should be blocked in a MediaInputPort.
  */
 enum class BlockingMode {
   /**
--- a/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
+++ b/media/webrtc/signaling/src/mediapipeline/MediaPipeline.cpp
@@ -1675,26 +1675,26 @@ public:
 
     mSource->AddTrackListener(this, mTrackId);
   }
 
   void AddSelf()
   {
     if (!mListening) {
       mListening = true;
-      mSource->SetPullEnabled(true);
+      mSource->SetPullingEnabled(mTrackId, true);
       mMaybeTrackNeedsUnmute = true;
     }
   }
 
   void RemoveSelf()
   {
     if (mListening) {
       mListening = false;
-      mSource->SetPullEnabled(false);
+      mSource->SetPullingEnabled(mTrackId, false);
     }
   }
 
   void OnRtpReceived()
   {
     if (mMaybeTrackNeedsUnmute) {
       mMaybeTrackNeedsUnmute = false;
       NS_DispatchToMainThread(