Bug 1221587: Base update of the MSG API for full-duplex r=padenot
authorRandell Jesup <rjesup@jesup.org>
Thu, 21 Jan 2016 11:51:35 -0500
changeset 281166 2d18cbea1800c23f06d78fc7c9c64503d2ab1ce3
parent 281165 93f83978cac6cc0368b80861c436f57337cbfd00
child 281167 269a82e6f643536d22564d5b88fb32148061e794
push id29930
push usercbook@mozilla.com
push dateFri, 22 Jan 2016 11:05:50 +0000
treeherdermozilla-central@7104d650a97d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerspadenot
bugs1221587
milestone46.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1221587: Base update of the MSG API for full-duplex r=padenot
dom/media/GraphDriver.cpp
dom/media/GraphDriver.h
dom/media/MediaStreamGraph.cpp
dom/media/MediaStreamGraph.h
dom/media/MediaStreamGraphImpl.h
--- a/dom/media/GraphDriver.cpp
+++ b/dom/media/GraphDriver.cpp
@@ -880,16 +880,33 @@ AudioCallbackDriver::DataCallback(AudioD
     stillProcessing = mGraphImpl->OneIteration(nextStateComputedTime);
   } else {
     NS_WARNING("DataCallback buffer filled entirely from scratch buffer, skipping iteration.");
     stillProcessing = true;
   }
 
   mBuffer.BufferFilled();
 
+  // Callback any observers for the AEC speaker data.  Note that one
+  // (maybe) of these will be full-duplex, the others will get their input
+  // data off separate cubeb callbacks.  Take care with how stuff is
+  // removed/added to this list and TSAN issues, but input and output will
+  // use separate callback methods.
+  mGraphImpl->NotifySpeakerData(aOutputBuffer, static_cast<size_t>(aFrames),
+                                ChannelCount);
+
+  // Process mic data if any/needed -- after inserting far-end data for AEC!
+  if (aInputBuffer) {
+    if (mAudioInput) { // for this specific input-only or full-duplex stream
+      mAudioInput->NotifyInputData(mGraphImpl, aInputBuffer,
+                                   static_cast<size_t>(aFrames),
+                                   ChannelCount);
+    }
+  }
+
   bool switching = false;
   {
     MonitorAutoLock mon(mGraphImpl->GetMonitor());
     switching = !!NextDriver();
   }
 
   if (switching && stillProcessing) {
     // If the audio stream has not been started by the previous driver or
--- a/dom/media/GraphDriver.h
+++ b/dom/media/GraphDriver.h
@@ -191,16 +191,26 @@ public:
   void EnsureNextIterationLocked();
 
   MediaStreamGraphImpl* GraphImpl() {
     return mGraphImpl;
   }
 
   virtual bool OnThread() = 0;
 
+  // XXX Thread-safety! Do these via commands to avoid TSAN issues
+  // and crashes!!!
+  virtual void SetInputListener(MediaStreamListener *aListener) {
+    mAudioInput = aListener;
+  }
+  // XXX do we need the param?  probably no
+  virtual void RemoveInputListener(MediaStreamListener *aListener) {
+    mAudioInput = nullptr;
+  }
+
 protected:
   GraphTime StateComputedTime() const;
 
   // Time of the start of this graph iteration. This must be accessed while
   // having the monitor.
   GraphTime mIterationStart;
   // Time of the end of this graph iteration. This must be accessed while having
   // the monitor.
@@ -221,16 +231,19 @@ protected:
     WAITSTATE_WAITING_INDEFINITELY,
     // Something has signaled RunThread() to wake up immediately,
     // but it hasn't done so yet
     WAITSTATE_WAKING_UP
   };
   // This must be access with the monitor.
   WaitState mWaitState;
 
+  // Callback for mic data, if any
+  RefPtr<MediaStreamListener> mAudioInput;
+
   // This is used on the main thread (during initialization), and the graph
   // thread. No monitor needed because we know the graph thread does not run
   // during the initialization.
   TimeStamp mCurrentTimeStamp;
   // This is non-null only when this driver has recently switched from an other
   // driver, and has not cleaned it up yet (for example because the audio stream
   // is currently calling the callback during initialization).
   //
@@ -483,16 +496,18 @@ private:
    * This is written on the previous driver's thread (if switching) or main
    * thread (if this driver is the first one).
    * This is read on previous driver's thread (during callbacks from
    * cubeb_stream_init) and the audio thread (when switching away from this
    * driver back to a SystemClockDriver).
    * This is synchronized by the Graph's monitor.
    * */
   bool mStarted;
+  /* Listener for mic input, if any. */
+  RefPtr<MediaStreamListener> mAudioInput;
 
   struct AutoInCallback
   {
     explicit AutoInCallback(AudioCallbackDriver* aDriver);
     ~AutoInCallback();
     AudioCallbackDriver* mDriver;
   };
 
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -25,16 +25,17 @@
 #include "mozilla/dom/AudioContextBinding.h"
 #include <algorithm>
 #include "DOMMediaStream.h"
 #include "GeckoProfiler.h"
 #include "mozilla/unused.h"
 #ifdef MOZ_WEBRTC
 #include "AudioOutputObserver.h"
 #endif
+#include "mtransport/runnable_utils.h"
 
 #include "webaudio/blink/HRTFDatabaseLoader.h"
 
 using namespace mozilla::layers;
 using namespace mozilla::dom;
 using namespace mozilla::gfx;
 
 namespace mozilla {
@@ -917,16 +918,95 @@ MediaStreamGraphImpl::PlayVideo(MediaStr
 
   // If the stream has finished and the timestamps of all frames have expired
   // then no more updates are required.
   if (aStream->mFinished && !haveMultipleImages) {
     aStream->mLastPlayedVideoFrame.SetNull();
   }
 }
 
+void
+MediaStreamGraphImpl::OpenAudioInputImpl(char *aName, MediaStreamListener *aListener)
+{
+  if (CurrentDriver()->AsAudioCallbackDriver()) {
+    CurrentDriver()->SetInputListener(aListener);
+  } else {
+    // XXX Switch to callback driver
+  }
+  mAudioInputs.AppendElement(aListener); // always monitor speaker data
+}
+
+nsresult
+MediaStreamGraphImpl::OpenAudioInput(char *aName, MediaStreamListener *aListener)
+{
+  // XXX So, so, so annoying.  Can't AppendMessage except on Mainthread
+  if (!NS_IsMainThread()) {
+    NS_DispatchToMainThread(WrapRunnable(this,
+                                         &MediaStreamGraphImpl::OpenAudioInput,
+                                         aName, aListener)); // XXX Fix! string need to copied
+    return NS_OK;
+  }
+  class Message : public ControlMessage {
+  public:
+    Message(MediaStreamGraphImpl *aGraph, char *aName, MediaStreamListener *aListener) :
+      ControlMessage(nullptr), mGraph(aGraph), mName(aName), mListener(aListener) {}
+    virtual void Run()
+    {
+      mGraph->OpenAudioInputImpl(mName, mListener);
+    }
+    MediaStreamGraphImpl *mGraph;
+    char *mName; // XXX needs to copy
+    MediaStreamListener *mListener;
+  };
+  this->AppendMessage(new Message(this, aName, aListener));
+  return NS_OK;
+}
+
+void
+MediaStreamGraphImpl::CloseAudioInputImpl(MediaStreamListener *aListener)
+{
+  CurrentDriver()->RemoveInputListener(aListener);
+  mAudioInputs.RemoveElement(aListener);
+}
+
+void
+MediaStreamGraphImpl::CloseAudioInput(MediaStreamListener *aListener)
+{
+  // XXX So, so, so annoying.  Can't AppendMessage except on Mainthread
+  if (!NS_IsMainThread()) {
+    NS_DispatchToMainThread(WrapRunnable(this,
+                                         &MediaStreamGraphImpl::CloseAudioInput,
+                                         aListener));
+    return;
+  }
+  class Message : public ControlMessage {
+  public:
+    Message(MediaStreamGraphImpl *aGraph, MediaStreamListener *aListener) :
+      ControlMessage(nullptr), mGraph(aGraph), mListener(aListener) {}
+    virtual void Run()
+    {
+      mGraph->CloseAudioInputImpl(mListener);
+    }
+    MediaStreamGraphImpl *mGraph;
+    MediaStreamListener *mListener;
+  };
+  this->AppendMessage(new Message(this, aListener));
+}
+
+
+// All AudioInput listeners get the same speaker data (at least for now).
+void
+MediaStreamGraph::NotifySpeakerData(AudioDataValue* aBuffer, size_t aFrames,
+                                    uint32_t aChannels)
+{
+  for (auto& listener : mAudioInputs) {
+    listener->NotifySpeakerData(this, aBuffer, aFrames, aChannels);
+  }
+}
+
 bool
 MediaStreamGraphImpl::ShouldUpdateMainThread()
 {
   if (mRealtime) {
     return true;
   }
 
   TimeStamp now = TimeStamp::Now();
--- a/dom/media/MediaStreamGraph.h
+++ b/dom/media/MediaStreamGraph.h
@@ -176,16 +176,33 @@ public:
                                         TrackID aInputTrackID = TRACK_INVALID) {}
 
   /**
    * Notify that all new tracks this iteration have been created.
    * This is to ensure that tracks added atomically to MediaStreamGraph
    * are also notified of atomically to MediaStreamListeners.
    */
   virtual void NotifyFinishedTrackCreation(MediaStreamGraph* aGraph) {}
+
+  /* These are for cubeb audio input & output streams: */
+  /**
+   * Output data to speakers, for use as the "far-end" data for echo
+   * cancellation.  This is not guaranteed to be in any particular size
+   * chunks.
+   */
+  virtual void NotifySpeakerData(MediaStreamGraph* aGraph,
+                                 AudioDataValue* aBuffer, size_t aFrames,
+                                 uint32_t aChannels) {}
+  /**
+   * Input data from a microphone (or other audio source.  This is not
+   * guaranteed to be in any particular size chunks.
+   */
+  virtual void NotifyInputData(MediaStreamGraph* aGraph,
+                               AudioDataValue* aBuffer, size_t aFrames,
+                               uint32_t aChannels) {}
 };
 
 /**
  * This is a base class for media graph thread listener direct callbacks
  * from within AppendToTrack().  Note that your regular listener will
  * still get NotifyQueuedTrackChanges() callbacks from the MSG thread, so
  * you must be careful to ignore them if AddDirectListener was successful.
  */
@@ -1170,16 +1187,21 @@ public:
   };
   // Main thread only
   static MediaStreamGraph* GetInstance(GraphDriverType aGraphDriverRequested,
                                        dom::AudioChannel aChannel);
   static MediaStreamGraph* CreateNonRealtimeInstance(TrackRate aSampleRate);
   // Idempotent
   static void DestroyNonRealtimeInstance(MediaStreamGraph* aGraph);
 
+  virtual nsresult OpenAudioInput(char *aName, MediaStreamListener *aListener) {
+    return NS_ERROR_FAILURE;
+  }
+  virtual void CloseAudioInput(MediaStreamListener *aListener) {}
+
   // Control API.
   /**
    * Create a stream that a media decoder (or some other source of
    * media data, such as a camera) can write to.
    */
   SourceMediaStream* CreateSourceStream(DOMMediaStream* aWrapper);
   /**
    * Create a stream that will form the union of the tracks of its input
@@ -1249,16 +1271,23 @@ public:
   TrackRate GraphRate() const { return mSampleRate; }
 
   void RegisterCaptureStreamForWindow(uint64_t aWindowId,
                                       ProcessedMediaStream* aCaptureStream);
   void UnregisterCaptureStreamForWindow(uint64_t aWindowId);
   already_AddRefed<MediaInputPort> ConnectToCaptureStream(
     uint64_t aWindowId, MediaStream* aMediaStream);
 
+  /**
+   * Data going to the speakers from the GraphDriver's DataCallback
+   * to notify any listeners (for echo cancellation).
+   */
+  void NotifySpeakerData(AudioDataValue* aBuffer, size_t aFrames,
+                         uint32_t aChannels);
+
 protected:
   explicit MediaStreamGraph(TrackRate aSampleRate)
     : mSampleRate(aSampleRate)
   {
     MOZ_COUNT_CTOR(MediaStreamGraph);
   }
   virtual ~MediaStreamGraph()
   {
@@ -1269,13 +1298,15 @@ protected:
   nsTArray<nsCOMPtr<nsIRunnable> > mPendingUpdateRunnables;
 
   /**
    * Sample rate at which this graph runs. For real time graphs, this is
    * the rate of the audio mixer. For offline graphs, this is the rate specified
    * at construction.
    */
   TrackRate mSampleRate;
+
+  nsTArray<RefPtr<MediaStreamListener>> mAudioInputs;
 };
 
 } // namespace mozilla
 
 #endif /* MOZILLA_MEDIASTREAMGRAPH_H_ */
--- a/dom/media/MediaStreamGraphImpl.h
+++ b/dom/media/MediaStreamGraphImpl.h
@@ -345,16 +345,21 @@ public:
    * Set the correct current video frame for stream aStream.
    */
   void PlayVideo(MediaStream* aStream);
   /**
    * No more data will be forthcoming for aStream. The stream will end
    * at the current buffer end point. The StreamBuffer's tracks must be
    * explicitly set to finished by the caller.
    */
+  void OpenAudioInputImpl(char *aName, MediaStreamListener *aListener);
+  virtual nsresult OpenAudioInput(char *aName, MediaStreamListener *aListener) override;
+  void CloseAudioInputImpl(MediaStreamListener *aListener);
+  virtual void CloseAudioInput(MediaStreamListener *aListener) override;
+
   void FinishStream(MediaStream* aStream);
   /**
    * Compute how much stream data we would like to buffer for aStream.
    */
   StreamTime GetDesiredBufferEnd(MediaStream* aStream);
   /**
    * Returns true when there are no active streams.
    */