Bug 1221587: patch 9 - Implement switching of AudioCallbackDrivers for full-duplex r?padenot draft
authorRandell Jesup <rjesup@jesup.org>
Fri, 08 Jan 2016 11:36:11 -0500
changeset 320108 f8d215bccfa4dd0ca0cc5d6afc994012e4029f4e
parent 320107 c418d693837d3090441d83799c1424e1df426890
child 320109 61c26d69c64da7c5d291ead0627fc6549d99a7d2
push id9134
push userrjesup@wgate.com
push dateFri, 08 Jan 2016 16:39:51 +0000
reviewerspadenot
bugs1221587
milestone46.0a1
Bug 1221587: patch 9 - Implement switching of AudioCallbackDrivers for full-duplex r?padenot
dom/media/GraphDriver.cpp
dom/media/GraphDriver.h
dom/media/MediaStreamGraph.cpp
--- a/dom/media/GraphDriver.cpp
+++ b/dom/media/GraphDriver.cpp
@@ -61,18 +61,24 @@ void GraphDriver::SetGraphTime(GraphDriv
   GraphImpl()->GetMonitor().AssertCurrentThreadOwns();
   // We set mIterationEnd here, because the first thing a driver do when it
   // does an iteration is to update graph times, so we are in fact setting
   // mIterationStart of the next iteration by setting the end of the previous
   // iteration.
   mIterationStart = aLastSwitchNextIterationStart;
   mIterationEnd = aLastSwitchNextIterationEnd;
 
-  STREAM_LOG(LogLevel::Debug, ("Setting previous driver: %p (%s)", PreviousDriver(), PreviousDriver()->AsAudioCallbackDriver() ? "AudioCallbackDriver" : "SystemClockDriver"));
   MOZ_ASSERT(!PreviousDriver());
+  MOZ_ASSERT(aPreviousDriver);
+
+  STREAM_LOG(LogLevel::Debug, ("Setting previous driver: %p (%s)",
+                               aPreviousDriver,
+                               aPreviousDriver->AsAudioCallbackDriver()
+                                 ? "AudioCallbackDriver"
+                                 : "SystemClockDriver"));
   SetPreviousDriver(aPreviousDriver);
 }
 
 void GraphDriver::SwitchAtNextIteration(GraphDriver* aNextDriver)
 {
   GraphImpl()->GetMonitor().AssertCurrentThreadOwns();
   // This is the situation where `mPreviousDriver` is an AudioCallbackDriver
   // that is switching device, and the graph has found the current driver is not
@@ -98,47 +104,16 @@ GraphDriver::StateComputedTime() const
   return mGraphImpl->mStateComputedTime;
 }
 
 void GraphDriver::EnsureNextIteration()
 {
   mGraphImpl->EnsureNextIteration();
 }
 
-class MediaStreamGraphShutdownThreadRunnable : public nsRunnable {
-public:
-  explicit MediaStreamGraphShutdownThreadRunnable(GraphDriver* aDriver)
-    : mDriver(aDriver)
-  {
-  }
-  NS_IMETHOD Run()
-  {
-    MOZ_ASSERT(NS_IsMainThread());
-
-    LIFECYCLE_LOG("MediaStreamGraphShutdownThreadRunnable for graph %p",
-        mDriver->GraphImpl());
-    // We can't release an audio driver on the main thread, because it can be
-    // blocking.
-    if (mDriver->AsAudioCallbackDriver()) {
-      LIFECYCLE_LOG("Releasing audio driver off main thread.");
-      RefPtr<AsyncCubebTask> releaseEvent =
-        new AsyncCubebTask(mDriver->AsAudioCallbackDriver(),
-                           AsyncCubebOperation::SHUTDOWN);
-      mDriver = nullptr;
-      releaseEvent->Dispatch();
-    } else {
-      LIFECYCLE_LOG("Dropping driver reference for SystemClockDriver.");
-      mDriver = nullptr;
-    }
-    return NS_OK;
-  }
-private:
-  RefPtr<GraphDriver> mDriver;
-};
-
 void GraphDriver::Shutdown()
 {
   if (AsAudioCallbackDriver()) {
     LIFECYCLE_LOG("Releasing audio driver off main thread (GraphDriver::Shutdown).\n");
     RefPtr<AsyncCubebTask> releaseEvent =
       new AsyncCubebTask(AsAudioCallbackDriver(), AsyncCubebOperation::SHUTDOWN);
     releaseEvent->Dispatch();
   } else {
@@ -537,16 +512,17 @@ StreamAndPromiseForOperation::StreamAndP
 
 AudioCallbackDriver::AudioCallbackDriver(MediaStreamGraphImpl* aGraphImpl)
   : GraphDriver(aGraphImpl)
   , mSampleRate(0)
   , mIterationDurationMS(MEDIA_GRAPH_TARGET_PERIOD_MS)
   , mStarted(false)
   , mAudioInput(nullptr)
   , mAudioChannel(aGraphImpl->AudioChannel())
+  , mAddedMixer(false)
   , mInCallback(false)
   , mMicrophoneActive(false)
 #ifdef XP_MACOSX
   , mCallbackReceivedWhileSwitching(0)
 #endif
 {
   STREAM_LOG(LogLevel::Debug, ("AudioCallbackDriver ctor for graph %p", aGraphImpl));
 }
@@ -644,45 +620,35 @@ AudioCallbackDriver::Resume()
   if (cubeb_stream_start(mAudioStream) != CUBEB_OK) {
     NS_WARNING("Could not start cubeb stream for MSG.");
   }
 }
 
 void
 AudioCallbackDriver::Start()
 {
-  // If this is running on the main thread, we can't open the stream directly,
-  // because it is a blocking operation.
-  if (NS_IsMainThread()) {
-    STREAM_LOG(LogLevel::Debug, ("Starting audio threads for MediaStreamGraph %p from a new thread.", mGraphImpl));
-    RefPtr<AsyncCubebTask> initEvent =
-      new AsyncCubebTask(this, AsyncCubebOperation::INIT);
-    initEvent->Dispatch();
-  } else {
-    STREAM_LOG(LogLevel::Debug, ("Starting audio threads for MediaStreamGraph %p from the previous driver's thread", mGraphImpl));
-    {
-      MonitorAutoUnlock mon(GraphImpl()->GetMonitor());
-      Init();
-    }
-
-    // Check if we need to resolve promises because the driver just got switched
-    // because of a resuming AudioContext
-    if (!mPromisesForOperation.IsEmpty()) {
-      // CompleteAudioContextOperations takes the lock as needed
-      MonitorAutoUnlock mon(GraphImpl()->GetMonitor());
-      CompleteAudioContextOperations(AsyncCubebOperation::INIT);
-    }
-
-    if (PreviousDriver()) {
-      nsCOMPtr<nsIRunnable> event =
-        new MediaStreamGraphShutdownThreadRunnable(PreviousDriver());
-      SetPreviousDriver(nullptr);
-      NS_DispatchToMainThread(event);
+  if (mPreviousDriver) {
+    if (mPreviousDriver->AsAudioCallbackDriver()) {
+      LIFECYCLE_LOG("Releasing audio driver off main thread.");
+      RefPtr<AsyncCubebTask> releaseEvent =
+        new AsyncCubebTask(mPreviousDriver->AsAudioCallbackDriver(),
+                           AsyncCubebOperation::SHUTDOWN);
+      releaseEvent->Dispatch();
+      mPreviousDriver = nullptr;
+    } else {
+      LIFECYCLE_LOG("Dropping driver reference for SystemClockDriver.");
+      mPreviousDriver = nullptr;
     }
   }
+
+  LIFECYCLE_LOG("Starting new audio driver off main thread, "
+                "to ensure it runs after previous shutdown.");
+  RefPtr<AsyncCubebTask> initEvent =
+    new AsyncCubebTask(AsAudioCallbackDriver(), AsyncCubebOperation::INIT);
+  initEvent->Dispatch();
 }
 
 void
 AudioCallbackDriver::StartStream()
 {
   if (cubeb_stream_start(mAudioStream) != CUBEB_OK) {
     MOZ_CRASH("Could not start cubeb stream for MSG.");
   }
@@ -807,16 +773,22 @@ AudioCallbackDriver::OSXDeviceSwitchingW
 #endif // XP_MACOSX
 
 long
 AudioCallbackDriver::DataCallback(AudioDataValue* aInputBuffer,
                                   AudioDataValue* aOutputBuffer, long aFrames)
 {
   bool stillProcessing;
 
+  // Don't add the callback until we're inited and ready
+  if (!mAddedMixer) {
+    mGraphImpl->mMixer.AddCallback(this);
+    mAddedMixer = true;
+  }
+
 #ifdef XP_MACOSX
   if (OSXDeviceSwitchingWorkaround()) {
     PodZero(aOutputBuffer, aFrames * mGraphImpl->AudioChannelCount());
     return aFrames;
   }
 #endif
 
 #ifdef DEBUG
@@ -888,17 +860,17 @@ AudioCallbackDriver::DataCallback(AudioD
 
     if (stateComputedTime < mIterationEnd) {
       STREAM_LOG(LogLevel::Warning, ("Media graph global underrun detected"));
       mIterationEnd = stateComputedTime;
     }
 
     stillProcessing = mGraphImpl->OneIteration(nextStateComputedTime);
   } else {
-    NS_WARNING("DataCallback buffer filled entirely from scratch buffer, skipping iteration.");
+    STREAM_LOG(LogLevel::Verbose, ("DataCallback buffer filled entirely from scratch buffer, skipping iteration."));
     stillProcessing = true;
   }
 
   mBuffer.BufferFilled();
 
   // Callback any observers for the AEC speaker data.  Note that one
   // (maybe) of these will be full-duplex, the others will get their input
   // data off separate cubeb callbacks.  Take care with how stuff is
--- a/dom/media/GraphDriver.h
+++ b/dom/media/GraphDriver.h
@@ -507,18 +507,21 @@ private:
     AudioCallbackDriver* mDriver;
   };
 
   /* Thread for off-main-thread initialization and
    * shutdown of the audio stream. */
   nsCOMPtr<nsIThread> mInitShutdownThread;
   /* This must be accessed with the graph monitor held. */
   nsAutoTArray<StreamAndPromiseForOperation, 1> mPromisesForOperation;
-  /* This is set during initialization, and ca be read safely afterwards. */
+  /* This is set during initialization, and can be read safely afterwards. */
   dom::AudioChannel mAudioChannel;
+  /* used to queue us to add the mixer callback on first run */
+  bool mAddedMixer;
+
   /* This is atomic and is set by the audio callback thread. It can be read by
    * any thread safely. */
   Atomic<bool> mInCallback;
   /**
    * True if microphone is being used by this process. This is synchronized by
    * the graph's monitor. */
   bool mMicrophoneActive;
 
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -369,17 +369,16 @@ MediaStreamGraphImpl::UpdateStreamOrder(
   }
 
   if (audioTrackPresent && mRealtime &&
       !CurrentDriver()->AsAudioCallbackDriver() &&
       !switching) {
     MonitorAutoLock mon(mMonitor);
     if (mLifecycleState == LIFECYCLE_RUNNING) {
       AudioCallbackDriver* driver = new AudioCallbackDriver(this);
-      mMixer.AddCallback(driver);
       CurrentDriver()->SwitchAtNextIteration(driver);
     }
   }
 
 #ifdef MOZ_WEBRTC
   if (shouldAEC && !mFarendObserverRef && gFarendObserver) {
     mFarendObserverRef = gFarendObserver;
     mMixer.AddCallback(mFarendObserverRef);
@@ -635,17 +634,16 @@ MediaStreamGraphImpl::CreateOrDestroyAud
         switching = CurrentDriver()->Switching();
       }
 
       if (!CurrentDriver()->AsAudioCallbackDriver() &&
           !switching) {
         MonitorAutoLock mon(mMonitor);
         if (mLifecycleState == LIFECYCLE_RUNNING) {
           AudioCallbackDriver* driver = new AudioCallbackDriver(this);
-          mMixer.AddCallback(driver);
           CurrentDriver()->SwitchAtNextIteration(driver);
         }
       }
     }
   }
 
   for (int32_t i = audioOutputStreamsFound.Length() - 1; i >= 0; --i) {
     if (!audioOutputStreamsFound[i]) {
@@ -921,68 +919,111 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
   if (aStream->mFinished && !haveMultipleImages) {
     aStream->mLastPlayedVideoFrame.SetNull();
   }
 }
 
 void
 MediaStreamGraphImpl::OpenAudioInputImpl(void *aID, AudioDataListener *aListener)
 {
+ // Bug XXXXXX Need support for multiple mics at once
   MOZ_ASSERT(!mInputWanted);
+  if (mInputWanted) {
+    // Need to support separate input-only AudioCallback drivers; they'll
+    // call us back on "other" threads.  We will need to echo-cancel them, though.
+    return;
+  }
   mInputWanted = true;
+  // aID is a cubeb_devid, and we assume that opaque ptr is valid until
+  // we close cubeb.
   mInputDeviceID = aID;
-  // XXX Switch Drivers
-  if (CurrentDriver()->AsAudioCallbackDriver()) {
-    CurrentDriver()->SetInputListener(aListener);
-  } else {
-    // XXX Switch to callback driver
+  mAudioInputs.AppendElement(aListener); // always monitor speaker data
+
+  // Switch Drivers since we're adding input (to input-only or full-duplex)
+  MonitorAutoLock mon(mMonitor);
+  if (mLifecycleState == LIFECYCLE_RUNNING) {
+    AudioCallbackDriver* driver = new AudioCallbackDriver(this);
+    mMixer.RemoveCallback(CurrentDriver()->AsAudioCallbackDriver());
+    CurrentDriver()->SwitchAtNextIteration(driver);
   }
-  mAudioInputs.AppendElement(aListener); // always monitor speaker data
 }
 
 nsresult
 MediaStreamGraphImpl::OpenAudioInput(void *aID, AudioDataListener *aListener)
 {
-  // XXX So, so, so annoying.  Can't AppendMessage except on Mainthread
+  // So, so, so annoying.  Can't AppendMessage except on Mainthread.
   if (!NS_IsMainThread()) {
     NS_DispatchToMainThread(WrapRunnable(this,
                                          &MediaStreamGraphImpl::OpenAudioInput,
-                                         aID, aListener)); // XXX Fix! string need to copied
+                                         aID, aListener));
     return NS_OK;
   }
   class Message : public ControlMessage {
   public:
     Message(MediaStreamGraphImpl *aGraph, void *aID, AudioDataListener *aListener) :
       ControlMessage(nullptr), mGraph(aGraph), mID(aID), mListener(aListener) {}
     virtual void Run()
     {
       mGraph->OpenAudioInputImpl(mID, mListener);
     }
     MediaStreamGraphImpl *mGraph;
-    void *mID; // XXX ownership?
+    // aID is a cubeb_devid, and we assume that opaque ptr is valid until
+    // we close cubeb.
+    void *mID;
     AudioDataListener *mListener;
   };
   this->AppendMessage(new Message(this, aID, aListener));
   return NS_OK;
 }
 
 void
 MediaStreamGraphImpl::CloseAudioInputImpl(AudioDataListener *aListener)
 {
   mInputDeviceID = nullptr;
   mInputWanted = false;
   CurrentDriver()->RemoveInputListener(aListener);
-  // XXX Switch Drivers
   mAudioInputs.RemoveElement(aListener);
+
+  // Switch Drivers since we're adding removing input (to nothing/system or output only)
+  bool audioTrackPresent = false;
+  for (uint32_t i = 0; i < mStreams.Length(); ++i) {
+    MediaStream* stream = mStreams[i];
+    // If this is a AudioNodeStream, force a AudioCallbackDriver.
+    if (stream->AsAudioNodeStream()) {
+      audioTrackPresent = true;
+    } else {
+      for (StreamBuffer::TrackIter tracks(stream->GetStreamBuffer(), MediaSegment::AUDIO);
+           !tracks.IsEnded(); tracks.Next()) {
+        audioTrackPresent = true;
+      }
+    }
+  }
+
+  MonitorAutoLock mon(mMonitor);
+  if (mLifecycleState == LIFECYCLE_RUNNING) {
+    GraphDriver* driver;
+    if (audioTrackPresent) {
+      // We still have audio output
+      STREAM_LOG(LogLevel::Debug, ("CloseInput: output present (AudioCallback)"));
+
+      driver = new AudioCallbackDriver(this);
+    } else {
+      STREAM_LOG(LogLevel::Debug, ("CloseInput: no output present (SystemClockCallback)"));
+
+      driver = new SystemClockDriver(this);
+      mMixer.RemoveCallback(CurrentDriver()->AsAudioCallbackDriver());
+    }
+    CurrentDriver()->SwitchAtNextIteration(driver);
+  }
 }
 
 void
 MediaStreamGraphImpl::CloseAudioInput(AudioDataListener *aListener)
 {
-  // XXX So, so, so annoying.  Can't AppendMessage except on Mainthread
+  // So, so, so annoying.  Can't AppendMessage except on Mainthread
   if (!NS_IsMainThread()) {
     NS_DispatchToMainThread(WrapRunnable(this,
                                          &MediaStreamGraphImpl::CloseAudioInput,
                                          aListener));
     return;
   }
   class Message : public ControlMessage {
   public:
@@ -2746,17 +2787,16 @@ MediaStreamGraphImpl::MediaStreamGraphIm
   , mCanRunMessagesSynchronously(false)
 #endif
   , mAudioChannel(aChannel)
 {
   if (mRealtime) {
     if (aDriverRequested == AUDIO_THREAD_DRIVER) {
       AudioCallbackDriver* driver = new AudioCallbackDriver(this);
       mDriver = driver;
-      mMixer.AddCallback(driver);
     } else {
       mDriver = new SystemClockDriver(this);
     }
   } else {
     mDriver = new OfflineClockDriver(this, MEDIA_GRAPH_TARGET_PERIOD_MS);
   }
 
   mLastMainThreadUpdate = TimeStamp::Now();
@@ -3152,17 +3192,16 @@ MediaStreamGraphImpl::ApplyAudioContextO
   if (aOperation == AudioContextOperation::Resume) {
     if (!CurrentDriver()->AsAudioCallbackDriver()) {
       AudioCallbackDriver* driver;
       if (switching) {
         MOZ_ASSERT(nextDriver->AsAudioCallbackDriver());
         driver = nextDriver->AsAudioCallbackDriver();
       } else {
         driver = new AudioCallbackDriver(this);
-        mMixer.AddCallback(driver);
         MonitorAutoLock lock(mMonitor);
         CurrentDriver()->SwitchAtNextIteration(driver);
       }
       driver->EnqueueStreamAndPromiseForOperation(aDestinationStream,
           aPromise, aOperation);
     } else {
       // We are resuming a context, but we are already using an
       // AudioCallbackDriver, we can resolve the promise now.