Bug 1221587: patch 9 - Implement switching of AudioCallbackDrivers for full-duplex r?padenot
--- a/dom/media/GraphDriver.cpp
+++ b/dom/media/GraphDriver.cpp
@@ -61,18 +61,24 @@ void GraphDriver::SetGraphTime(GraphDriv
GraphImpl()->GetMonitor().AssertCurrentThreadOwns();
// We set mIterationEnd here, because the first thing a driver do when it
// does an iteration is to update graph times, so we are in fact setting
// mIterationStart of the next iteration by setting the end of the previous
// iteration.
mIterationStart = aLastSwitchNextIterationStart;
mIterationEnd = aLastSwitchNextIterationEnd;
- STREAM_LOG(LogLevel::Debug, ("Setting previous driver: %p (%s)", PreviousDriver(), PreviousDriver()->AsAudioCallbackDriver() ? "AudioCallbackDriver" : "SystemClockDriver"));
MOZ_ASSERT(!PreviousDriver());
+ MOZ_ASSERT(aPreviousDriver);
+
+ STREAM_LOG(LogLevel::Debug, ("Setting previous driver: %p (%s)",
+ aPreviousDriver,
+ aPreviousDriver->AsAudioCallbackDriver()
+ ? "AudioCallbackDriver"
+ : "SystemClockDriver"));
SetPreviousDriver(aPreviousDriver);
}
void GraphDriver::SwitchAtNextIteration(GraphDriver* aNextDriver)
{
GraphImpl()->GetMonitor().AssertCurrentThreadOwns();
// This is the situation where `mPreviousDriver` is an AudioCallbackDriver
// that is switching device, and the graph has found the current driver is not
@@ -98,47 +104,16 @@ GraphDriver::StateComputedTime() const
return mGraphImpl->mStateComputedTime;
}
void GraphDriver::EnsureNextIteration()
{
mGraphImpl->EnsureNextIteration();
}
-class MediaStreamGraphShutdownThreadRunnable : public nsRunnable {
-public:
- explicit MediaStreamGraphShutdownThreadRunnable(GraphDriver* aDriver)
- : mDriver(aDriver)
- {
- }
- NS_IMETHOD Run()
- {
- MOZ_ASSERT(NS_IsMainThread());
-
- LIFECYCLE_LOG("MediaStreamGraphShutdownThreadRunnable for graph %p",
- mDriver->GraphImpl());
- // We can't release an audio driver on the main thread, because it can be
- // blocking.
- if (mDriver->AsAudioCallbackDriver()) {
- LIFECYCLE_LOG("Releasing audio driver off main thread.");
- RefPtr<AsyncCubebTask> releaseEvent =
- new AsyncCubebTask(mDriver->AsAudioCallbackDriver(),
- AsyncCubebOperation::SHUTDOWN);
- mDriver = nullptr;
- releaseEvent->Dispatch();
- } else {
- LIFECYCLE_LOG("Dropping driver reference for SystemClockDriver.");
- mDriver = nullptr;
- }
- return NS_OK;
- }
-private:
- RefPtr<GraphDriver> mDriver;
-};
-
void GraphDriver::Shutdown()
{
if (AsAudioCallbackDriver()) {
LIFECYCLE_LOG("Releasing audio driver off main thread (GraphDriver::Shutdown).\n");
RefPtr<AsyncCubebTask> releaseEvent =
new AsyncCubebTask(AsAudioCallbackDriver(), AsyncCubebOperation::SHUTDOWN);
releaseEvent->Dispatch();
} else {
@@ -537,16 +512,17 @@ StreamAndPromiseForOperation::StreamAndP
AudioCallbackDriver::AudioCallbackDriver(MediaStreamGraphImpl* aGraphImpl)
: GraphDriver(aGraphImpl)
, mSampleRate(0)
, mIterationDurationMS(MEDIA_GRAPH_TARGET_PERIOD_MS)
, mStarted(false)
, mAudioInput(nullptr)
, mAudioChannel(aGraphImpl->AudioChannel())
+ , mAddedMixer(false)
, mInCallback(false)
, mMicrophoneActive(false)
#ifdef XP_MACOSX
, mCallbackReceivedWhileSwitching(0)
#endif
{
STREAM_LOG(LogLevel::Debug, ("AudioCallbackDriver ctor for graph %p", aGraphImpl));
}
@@ -644,45 +620,35 @@ AudioCallbackDriver::Resume()
if (cubeb_stream_start(mAudioStream) != CUBEB_OK) {
NS_WARNING("Could not start cubeb stream for MSG.");
}
}
void
AudioCallbackDriver::Start()
{
- // If this is running on the main thread, we can't open the stream directly,
- // because it is a blocking operation.
- if (NS_IsMainThread()) {
- STREAM_LOG(LogLevel::Debug, ("Starting audio threads for MediaStreamGraph %p from a new thread.", mGraphImpl));
- RefPtr<AsyncCubebTask> initEvent =
- new AsyncCubebTask(this, AsyncCubebOperation::INIT);
- initEvent->Dispatch();
- } else {
- STREAM_LOG(LogLevel::Debug, ("Starting audio threads for MediaStreamGraph %p from the previous driver's thread", mGraphImpl));
- {
- MonitorAutoUnlock mon(GraphImpl()->GetMonitor());
- Init();
- }
-
- // Check if we need to resolve promises because the driver just got switched
- // because of a resuming AudioContext
- if (!mPromisesForOperation.IsEmpty()) {
- // CompleteAudioContextOperations takes the lock as needed
- MonitorAutoUnlock mon(GraphImpl()->GetMonitor());
- CompleteAudioContextOperations(AsyncCubebOperation::INIT);
- }
-
- if (PreviousDriver()) {
- nsCOMPtr<nsIRunnable> event =
- new MediaStreamGraphShutdownThreadRunnable(PreviousDriver());
- SetPreviousDriver(nullptr);
- NS_DispatchToMainThread(event);
+ if (mPreviousDriver) {
+ if (mPreviousDriver->AsAudioCallbackDriver()) {
+ LIFECYCLE_LOG("Releasing audio driver off main thread.");
+ RefPtr<AsyncCubebTask> releaseEvent =
+ new AsyncCubebTask(mPreviousDriver->AsAudioCallbackDriver(),
+ AsyncCubebOperation::SHUTDOWN);
+ releaseEvent->Dispatch();
+ mPreviousDriver = nullptr;
+ } else {
+ LIFECYCLE_LOG("Dropping driver reference for SystemClockDriver.");
+ mPreviousDriver = nullptr;
}
}
+
+ LIFECYCLE_LOG("Starting new audio driver off main thread, "
+ "to ensure it runs after previous shutdown.");
+ RefPtr<AsyncCubebTask> initEvent =
+ new AsyncCubebTask(AsAudioCallbackDriver(), AsyncCubebOperation::INIT);
+ initEvent->Dispatch();
}
void
AudioCallbackDriver::StartStream()
{
if (cubeb_stream_start(mAudioStream) != CUBEB_OK) {
MOZ_CRASH("Could not start cubeb stream for MSG.");
}
@@ -807,16 +773,22 @@ AudioCallbackDriver::OSXDeviceSwitchingW
#endif // XP_MACOSX
long
AudioCallbackDriver::DataCallback(AudioDataValue* aInputBuffer,
AudioDataValue* aOutputBuffer, long aFrames)
{
bool stillProcessing;
+ // Don't add the callback until we're inited and ready
+ if (!mAddedMixer) {
+ mGraphImpl->mMixer.AddCallback(this);
+ mAddedMixer = true;
+ }
+
#ifdef XP_MACOSX
if (OSXDeviceSwitchingWorkaround()) {
PodZero(aOutputBuffer, aFrames * mGraphImpl->AudioChannelCount());
return aFrames;
}
#endif
#ifdef DEBUG
@@ -888,17 +860,17 @@ AudioCallbackDriver::DataCallback(AudioD
if (stateComputedTime < mIterationEnd) {
STREAM_LOG(LogLevel::Warning, ("Media graph global underrun detected"));
mIterationEnd = stateComputedTime;
}
stillProcessing = mGraphImpl->OneIteration(nextStateComputedTime);
} else {
- NS_WARNING("DataCallback buffer filled entirely from scratch buffer, skipping iteration.");
+ STREAM_LOG(LogLevel::Verbose, ("DataCallback buffer filled entirely from scratch buffer, skipping iteration."));
stillProcessing = true;
}
mBuffer.BufferFilled();
// Callback any observers for the AEC speaker data. Note that one
// (maybe) of these will be full-duplex, the others will get their input
// data off separate cubeb callbacks. Take care with how stuff is
--- a/dom/media/GraphDriver.h
+++ b/dom/media/GraphDriver.h
@@ -507,18 +507,21 @@ private:
AudioCallbackDriver* mDriver;
};
/* Thread for off-main-thread initialization and
* shutdown of the audio stream. */
nsCOMPtr<nsIThread> mInitShutdownThread;
/* This must be accessed with the graph monitor held. */
nsAutoTArray<StreamAndPromiseForOperation, 1> mPromisesForOperation;
- /* This is set during initialization, and ca be read safely afterwards. */
+ /* This is set during initialization, and can be read safely afterwards. */
dom::AudioChannel mAudioChannel;
+ /* used to queue us to add the mixer callback on first run */
+ bool mAddedMixer;
+
/* This is atomic and is set by the audio callback thread. It can be read by
* any thread safely. */
Atomic<bool> mInCallback;
/**
* True if microphone is being used by this process. This is synchronized by
* the graph's monitor. */
bool mMicrophoneActive;
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -369,17 +369,16 @@ MediaStreamGraphImpl::UpdateStreamOrder(
}
if (audioTrackPresent && mRealtime &&
!CurrentDriver()->AsAudioCallbackDriver() &&
!switching) {
MonitorAutoLock mon(mMonitor);
if (mLifecycleState == LIFECYCLE_RUNNING) {
AudioCallbackDriver* driver = new AudioCallbackDriver(this);
- mMixer.AddCallback(driver);
CurrentDriver()->SwitchAtNextIteration(driver);
}
}
#ifdef MOZ_WEBRTC
if (shouldAEC && !mFarendObserverRef && gFarendObserver) {
mFarendObserverRef = gFarendObserver;
mMixer.AddCallback(mFarendObserverRef);
@@ -635,17 +634,16 @@ MediaStreamGraphImpl::CreateOrDestroyAud
switching = CurrentDriver()->Switching();
}
if (!CurrentDriver()->AsAudioCallbackDriver() &&
!switching) {
MonitorAutoLock mon(mMonitor);
if (mLifecycleState == LIFECYCLE_RUNNING) {
AudioCallbackDriver* driver = new AudioCallbackDriver(this);
- mMixer.AddCallback(driver);
CurrentDriver()->SwitchAtNextIteration(driver);
}
}
}
}
for (int32_t i = audioOutputStreamsFound.Length() - 1; i >= 0; --i) {
if (!audioOutputStreamsFound[i]) {
@@ -921,68 +919,111 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
if (aStream->mFinished && !haveMultipleImages) {
aStream->mLastPlayedVideoFrame.SetNull();
}
}
void
MediaStreamGraphImpl::OpenAudioInputImpl(void *aID, AudioDataListener *aListener)
{
+ // Bug XXXXXX Need support for multiple mics at once
MOZ_ASSERT(!mInputWanted);
+ if (mInputWanted) {
+ // Need to support separate input-only AudioCallback drivers; they'll
+ // call us back on "other" threads. We will need to echo-cancel them, though.
+ return;
+ }
mInputWanted = true;
+ // aID is a cubeb_devid, and we assume that opaque ptr is valid until
+ // we close cubeb.
mInputDeviceID = aID;
- // XXX Switch Drivers
- if (CurrentDriver()->AsAudioCallbackDriver()) {
- CurrentDriver()->SetInputListener(aListener);
- } else {
- // XXX Switch to callback driver
+ mAudioInputs.AppendElement(aListener); // always monitor speaker data
+
+ // Switch Drivers since we're adding input (to input-only or full-duplex)
+ MonitorAutoLock mon(mMonitor);
+ if (mLifecycleState == LIFECYCLE_RUNNING) {
+ AudioCallbackDriver* driver = new AudioCallbackDriver(this);
+ mMixer.RemoveCallback(CurrentDriver()->AsAudioCallbackDriver());
+ CurrentDriver()->SwitchAtNextIteration(driver);
}
- mAudioInputs.AppendElement(aListener); // always monitor speaker data
}
nsresult
MediaStreamGraphImpl::OpenAudioInput(void *aID, AudioDataListener *aListener)
{
- // XXX So, so, so annoying. Can't AppendMessage except on Mainthread
+ // So, so, so annoying. Can't AppendMessage except on Mainthread.
if (!NS_IsMainThread()) {
NS_DispatchToMainThread(WrapRunnable(this,
&MediaStreamGraphImpl::OpenAudioInput,
- aID, aListener)); // XXX Fix! string need to copied
+ aID, aListener));
return NS_OK;
}
class Message : public ControlMessage {
public:
Message(MediaStreamGraphImpl *aGraph, void *aID, AudioDataListener *aListener) :
ControlMessage(nullptr), mGraph(aGraph), mID(aID), mListener(aListener) {}
virtual void Run()
{
mGraph->OpenAudioInputImpl(mID, mListener);
}
MediaStreamGraphImpl *mGraph;
- void *mID; // XXX ownership?
+ // aID is a cubeb_devid, and we assume that opaque ptr is valid until
+ // we close cubeb.
+ void *mID;
AudioDataListener *mListener;
};
this->AppendMessage(new Message(this, aID, aListener));
return NS_OK;
}
void
MediaStreamGraphImpl::CloseAudioInputImpl(AudioDataListener *aListener)
{
mInputDeviceID = nullptr;
mInputWanted = false;
CurrentDriver()->RemoveInputListener(aListener);
- // XXX Switch Drivers
mAudioInputs.RemoveElement(aListener);
+
+ // Switch Drivers since we're adding removing input (to nothing/system or output only)
+ bool audioTrackPresent = false;
+ for (uint32_t i = 0; i < mStreams.Length(); ++i) {
+ MediaStream* stream = mStreams[i];
+ // If this is a AudioNodeStream, force a AudioCallbackDriver.
+ if (stream->AsAudioNodeStream()) {
+ audioTrackPresent = true;
+ } else {
+ for (StreamBuffer::TrackIter tracks(stream->GetStreamBuffer(), MediaSegment::AUDIO);
+ !tracks.IsEnded(); tracks.Next()) {
+ audioTrackPresent = true;
+ }
+ }
+ }
+
+ MonitorAutoLock mon(mMonitor);
+ if (mLifecycleState == LIFECYCLE_RUNNING) {
+ GraphDriver* driver;
+ if (audioTrackPresent) {
+ // We still have audio output
+ STREAM_LOG(LogLevel::Debug, ("CloseInput: output present (AudioCallback)"));
+
+ driver = new AudioCallbackDriver(this);
+ } else {
+ STREAM_LOG(LogLevel::Debug, ("CloseInput: no output present (SystemClockCallback)"));
+
+ driver = new SystemClockDriver(this);
+ mMixer.RemoveCallback(CurrentDriver()->AsAudioCallbackDriver());
+ }
+ CurrentDriver()->SwitchAtNextIteration(driver);
+ }
}
void
MediaStreamGraphImpl::CloseAudioInput(AudioDataListener *aListener)
{
- // XXX So, so, so annoying. Can't AppendMessage except on Mainthread
+ // So, so, so annoying. Can't AppendMessage except on Mainthread
if (!NS_IsMainThread()) {
NS_DispatchToMainThread(WrapRunnable(this,
&MediaStreamGraphImpl::CloseAudioInput,
aListener));
return;
}
class Message : public ControlMessage {
public:
@@ -2746,17 +2787,16 @@ MediaStreamGraphImpl::MediaStreamGraphIm
, mCanRunMessagesSynchronously(false)
#endif
, mAudioChannel(aChannel)
{
if (mRealtime) {
if (aDriverRequested == AUDIO_THREAD_DRIVER) {
AudioCallbackDriver* driver = new AudioCallbackDriver(this);
mDriver = driver;
- mMixer.AddCallback(driver);
} else {
mDriver = new SystemClockDriver(this);
}
} else {
mDriver = new OfflineClockDriver(this, MEDIA_GRAPH_TARGET_PERIOD_MS);
}
mLastMainThreadUpdate = TimeStamp::Now();
@@ -3152,17 +3192,16 @@ MediaStreamGraphImpl::ApplyAudioContextO
if (aOperation == AudioContextOperation::Resume) {
if (!CurrentDriver()->AsAudioCallbackDriver()) {
AudioCallbackDriver* driver;
if (switching) {
MOZ_ASSERT(nextDriver->AsAudioCallbackDriver());
driver = nextDriver->AsAudioCallbackDriver();
} else {
driver = new AudioCallbackDriver(this);
- mMixer.AddCallback(driver);
MonitorAutoLock lock(mMonitor);
CurrentDriver()->SwitchAtNextIteration(driver);
}
driver->EnqueueStreamAndPromiseForOperation(aDestinationStream,
aPromise, aOperation);
} else {
// We are resuming a context, but we are already using an
// AudioCallbackDriver, we can resolve the promise now.