Bug 1397793 - Move to APM - Part 1 - UpdateSingleSource. r=pehrsons
authorPaul Adenot <paul@paul.cx>
Tue, 31 Oct 2017 18:25:41 +0100
changeset 396335 0ec5d3fdaf1bd40f1aeac1d697738d9b2626e8c3
parent 396334 d30ce07c556701f8729cf37a1b9afeb2705b36b1
child 396336 2b871bc4a4ba59b94dfb9319b7e6debe91022320
push id56975
push userdluca@mozilla.com
push dateThu, 14 Dec 2017 09:59:07 +0000
treeherderautoland@16bcfaad13e1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerspehrsons
bugs1397793
milestone59.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1397793 - Move to APM - Part 1 - UpdateSingleSource. r=pehrsons This part is about setting on/off audio processing feature. It's long, but it's mostly mechanichal changes, from the old API to the new one. This also covers reseting the processing in case of device changes (with macros). MozReview-Commit-ID: EI2TxHRicEr
dom/media/webrtc/MediaEngineWebRTC.h
dom/media/webrtc/MediaEngineWebRTCAudio.cpp
modules/libpref/init/all.js
--- a/dom/media/webrtc/MediaEngineWebRTC.h
+++ b/dom/media/webrtc/MediaEngineWebRTC.h
@@ -508,19 +508,20 @@ private:
 
   RefPtr<mozilla::AudioInput> mAudioInput;
   RefPtr<WebRTCAudioDataListener> mListener;
   RefPtr<AudioOutputObserver> mAudioOutputObserver;
 
   // Note: shared across all microphone sources
   static int sChannelsOpen;
 
+  const UniquePtr<webrtc::AudioProcessing> mAudioProcessing;
 
   // accessed from the GraphDriver thread except for deletion
-  nsAutoPtr<AudioPacketizer<AudioDataValue, int16_t>> mPacketizer;
+  nsAutoPtr<AudioPacketizer<AudioDataValue, AudioDataValue>> mPacketizer;
   ScopedCustomReleasePtr<webrtc::VoEExternalMedia> mVoERenderListener;
 
   // mMonitor protects mSources[] and mPrinicpalIds[] access/changes, and
   // transitions of mState from kStarted to kStopped (which are combined with
   // EndTrack()). mSources[] and mPrincipalHandles[] are accessed from webrtc
   // threads.
   Monitor mMonitor;
   nsTArray<RefPtr<SourceMediaStream>> mSources;
@@ -534,27 +535,26 @@ private:
 
   nsString mDeviceName;
   nsCString mDeviceUUID;
 
   int32_t mSampleFrequency;
   uint64_t mTotalFrames;
   uint64_t mLastLogFrames;
 
-  NullTransport *mNullTransport;
-
   // mSkipProcessing is true if none of the processing passes are enabled,
   // because of prefs or constraints. This allows simply copying the audio into
   // the MSG, skipping resampling and the whole webrtc.org code.
   bool mSkipProcessing;
 
   // To only update microphone when needed, we keep track of previous settings.
   MediaEnginePrefs mLastPrefs;
 
-  AlignedShortBuffer mInputDownmixBuffer;
+  AlignedFloatBuffer mInputBuffer;
+  AlignedFloatBuffer mInputDownmixBuffer;
 };
 
 class MediaEngineWebRTC : public MediaEngine
 {
   typedef MediaEngine Super;
 public:
   explicit MediaEngineWebRTC(MediaEnginePrefs& aPrefs);
 
--- a/dom/media/webrtc/MediaEngineWebRTCAudio.cpp
+++ b/dom/media/webrtc/MediaEngineWebRTCAudio.cpp
@@ -11,16 +11,21 @@
 #include "nsAutoPtr.h"
 #include "AudioConverter.h"
 
 // scoped_ptr.h uses FF
 #ifdef FF
 #undef FF
 #endif
 #include "webrtc/modules/audio_device/opensl/single_rw_fifo.h"
+#include "webrtc/voice_engine/voice_engine_defines.h"
+#include "webrtc/modules/audio_processing/include/audio_processing.h"
+#include "webrtc/common_audio/include/audio_util.h"
+
+using namespace webrtc;
 
 #define CHANNELS 1
 #define ENCODING "L16"
 #define DEFAULT_PORT 5555
 
 #define SAMPLE_RATE(freq) ((freq)*2*8) // bps, 16-bit samples
 #define SAMPLE_LENGTH(freq) (((freq)*10)/1000)
 
@@ -58,17 +63,17 @@ AudioOutputObserver::AudioOutputObserver
   : mPlayoutFreq(0)
   , mPlayoutChannels(0)
   , mChunkSize(0)
   , mSaved(nullptr)
   , mSamplesSaved(0)
   , mDownmixBuffer(MAX_SAMPLING_FREQ * MAX_CHANNELS / 100)
 {
   // Buffers of 10ms chunks
-  mPlayoutFifo = new webrtc::SingleRwFifo(MAX_AEC_FIFO_DEPTH/10);
+  mPlayoutFifo = new SingleRwFifo(MAX_AEC_FIFO_DEPTH/10);
 }
 
 AudioOutputObserver::~AudioOutputObserver()
 {
   Clear();
   free(mSaved);
   mSaved = nullptr;
 }
@@ -186,16 +191,17 @@ MediaEngineWebRTCMicrophoneSource::Media
     mozilla::AudioInput* aAudioInput,
     int aIndex,
     const char* name,
     const char* uuid,
     bool aDelayAgnostic,
     bool aExtendedFilter)
   : MediaEngineAudioSource(kReleased)
   , mAudioInput(aAudioInput)
+  , mAudioProcessing(AudioProcessing::Create())
   , mMonitor("WebRTCMic.Monitor")
   , mCapIndex(aIndex)
   , mDelayAgnostic(aDelayAgnostic)
   , mExtendedFilter(aExtendedFilter)
   , mTrackID(TRACK_NONE)
   , mStarted(false)
   , mSampleFrequency(MediaEngine::DEFAULT_SAMPLE_RATE)
   , mTotalFrames(0)
@@ -261,16 +267,150 @@ MediaEngineWebRTCMicrophoneSource::Resta
                               aOutBadConstraint);
 }
 
 bool operator == (const MediaEnginePrefs& a, const MediaEnginePrefs& b)
 {
   return !memcmp(&a, &b, sizeof(MediaEnginePrefs));
 };
 
+// This does an early return in case of error.
+#define HANDLE_APM_ERROR(fn)                                \
+do {                                                        \
+  int rv = fn;                                              \
+  if (rv != AudioProcessing::kNoError) {                    \
+    MOZ_ASSERT_UNREACHABLE("APM error in " #fn);            \
+    return;                                                 \
+  }                                                         \
+} while(0);
+
+void MediaEngineWebRTCMicrophoneSource::UpdateAECSettingsIfNeeded(bool aEnable, EcModes aMode)
+{
+  using webrtc::EcModes;
+
+  EchoCancellation::SuppressionLevel level;
+
+  switch(aMode) {
+    case EcModes::kEcUnchanged:
+      level = mAudioProcessing->echo_cancellation()->suppression_level();
+      break;
+    case EcModes::kEcConference:
+      level = EchoCancellation::kHighSuppression;
+      break;
+    case EcModes::kEcDefault:
+      level = EchoCancellation::kModerateSuppression;
+      break;
+    case EcModes::kEcAec:
+      level = EchoCancellation::kModerateSuppression;
+      break;
+    case EcModes::kEcAecm:
+      // No suppression level to set for the mobile echo canceller
+      break;
+    default:
+      MOZ_LOG(GetMediaManagerLog(), LogLevel::Error, ("Bad EcMode value"));
+      MOZ_ASSERT_UNREACHABLE("Bad pref set in all.js or in about:config"
+                             " for the echo cancelation mode.");
+      // fall back to something sensible in release
+      level = EchoCancellation::kModerateSuppression;
+      break;
+  }
+
+  // AECm and AEC are mutually exclusive.
+  if (aMode == EcModes::kEcAecm) {
+    HANDLE_APM_ERROR(mAudioProcessing->echo_cancellation()->Enable(false));
+    HANDLE_APM_ERROR(mAudioProcessing->echo_control_mobile()->Enable(aEnable));
+  } else {
+    HANDLE_APM_ERROR(mAudioProcessing->echo_control_mobile()->Enable(false));
+    HANDLE_APM_ERROR(mAudioProcessing->echo_cancellation()->Enable(aEnable));
+    HANDLE_APM_ERROR(mAudioProcessing->echo_cancellation()->set_suppression_level(level));
+  }
+}
+
+void
+MediaEngineWebRTCMicrophoneSource::UpdateAGCSettingsIfNeeded(bool aEnable, AgcModes aMode)
+{
+#if defined(WEBRTC_IOS) || defined(ATA) || defined(WEBRTC_ANDROID)
+  if (aMode == kAgcAdaptiveAnalog) {
+    MOZ_LOG(GetMediaManagerLog(),
+            LogLevel::Error,
+            ("Invalid AGC mode kAgcAdaptiveAnalog on mobile"));
+    MOZ_ASSERT_UNREACHABLE("Bad pref set in all.js or in about:config"
+                           " for the auto gain, on mobile.");
+    aMode = kAgcDefault;
+  }
+#endif
+  GainControl::Mode mode = kDefaultAgcMode;
+
+  switch (aMode) {
+    case AgcModes::kAgcDefault:
+      mode = kDefaultAgcMode;
+      break;
+    case AgcModes::kAgcUnchanged:
+      mode = mAudioProcessing->gain_control()->mode();
+      break;
+    case AgcModes::kAgcFixedDigital:
+      mode = GainControl::Mode::kFixedDigital;
+      break;
+    case AgcModes::kAgcAdaptiveAnalog:
+      mode = GainControl::Mode::kAdaptiveAnalog;
+      break;
+    case AgcModes::kAgcAdaptiveDigital:
+      mode = GainControl::Mode::kAdaptiveDigital;
+      break;
+    default:
+      MOZ_ASSERT_UNREACHABLE("Bad pref set in all.js or in about:config"
+                             " for the auto gain.");
+      // This is a good fallback, it works regardless of the platform.
+      mode = GainControl::Mode::kAdaptiveDigital;
+      break;
+  }
+
+  HANDLE_APM_ERROR(mAudioProcessing->gain_control()->set_mode(mode));
+  HANDLE_APM_ERROR(mAudioProcessing->gain_control()->Enable(aEnable));
+}
+
+void
+MediaEngineWebRTCMicrophoneSource::UpdateNSSettingsIfNeeded(bool aEnable, NsModes aMode)
+{
+  NoiseSuppression::Level nsLevel;
+
+  switch (aMode) {
+    case NsModes::kNsDefault:
+      nsLevel = kDefaultNsMode;
+      break;
+    case NsModes::kNsUnchanged:
+      nsLevel = mAudioProcessing->noise_suppression()->level();
+      break;
+    case NsModes::kNsConference:
+      nsLevel = NoiseSuppression::kHigh;
+      break;
+    case NsModes::kNsLowSuppression:
+      nsLevel = NoiseSuppression::kLow;
+      break;
+    case NsModes::kNsModerateSuppression:
+      nsLevel = NoiseSuppression::kModerate;
+      break;
+    case NsModes::kNsHighSuppression:
+      nsLevel = NoiseSuppression::kHigh;
+      break;
+    case NsModes::kNsVeryHighSuppression:
+      nsLevel = NoiseSuppression::kVeryHigh;
+      break;
+    default:
+      MOZ_ASSERT_UNREACHABLE("Bad pref set in all.js or in about:config"
+                             " for the noise suppression.");
+      // Pick something sensible as a faillback in release.
+      nsLevel = NoiseSuppression::kModerate;
+  }
+  HANDLE_APM_ERROR(mAudioProcessing->noise_suppression()->set_level(nsLevel));
+  HANDLE_APM_ERROR(mAudioProcessing->noise_suppression()->Enable(aEnable));
+}
+
+#undef HANDLE_APM_ERROR
+
 nsresult
 MediaEngineWebRTCMicrophoneSource::UpdateSingleSource(
     const AllocationHandle* aHandle,
     const NormalizedConstraints& aNetConstraints,
     const NormalizedConstraints& aNewConstraint, /* Ignored */
     const MediaEnginePrefs& aPrefs,
     const nsString& aDeviceId,
     const char** aOutBadConstraint)
@@ -309,56 +449,44 @@ MediaEngineWebRTCMicrophoneSource::Updat
   switch (mState) {
     case kReleased:
       MOZ_ASSERT(aHandle);
       if (sChannelsOpen != 0) {
         // Until we fix (or wallpaper) support for multiple mic input
         // (Bug 1238038) fail allocation for a second device
         return NS_ERROR_FAILURE;
       }
-      if (mAudioInput->SetRecordingDevice(mCapIndex)) {
-         return NS_ERROR_FAILURE;
-      }
       mAudioInput->SetUserChannelCount(prefs.mChannels);
       if (!AllocChannel()) {
         FreeChannel();
         LOG(("Audio device is not initalized"));
         return NS_ERROR_FAILURE;
       }
-      LOG(("Audio device %d allocated", mCapIndex));
-      {
-        // Update with the actual applied channelCount in order
-        // to store it in settings.
-        uint32_t channelCount = 0;
-        mAudioInput->GetChannelCount(channelCount);
-        MOZ_ASSERT(channelCount > 0);
-        prefs.mChannels = channelCount;
-      }
       break;
 
     case kStarted:
       if (prefs == mLastPrefs) {
         return NS_OK;
       }
 
       if (prefs.mChannels != mLastPrefs.mChannels) {
         MOZ_ASSERT(mSources.Length() > 0);
+        // If the channel count changed, tell the MSG to open a new driver with
+        // the correct channel count.
         auto& source = mSources.LastElement();
         mAudioInput->SetUserChannelCount(prefs.mChannels);
         // Get validated number of channel
         uint32_t channelCount = 0;
         mAudioInput->GetChannelCount(channelCount);
         MOZ_ASSERT(channelCount > 0 && mLastPrefs.mChannels > 0);
-        // Check if new validated channels is the same as previous
-        if (static_cast<uint32_t>(mLastPrefs.mChannels) != channelCount &&
+        if (mLastPrefs.mChannels != prefs.mChannels &&
             !source->OpenNewAudioCallbackDriver(mListener)) {
+          MOZ_LOG(GetMediaManagerLog(), LogLevel::Error, ("Could not open a new AudioCallbackDriver for input"));
           return NS_ERROR_FAILURE;
         }
-        // Update settings
-        prefs.mChannels = channelCount;
       }
 
       if (MOZ_LOG_TEST(GetMediaManagerLog(), LogLevel::Debug)) {
         MonitorAutoLock lock(mMonitor);
         if (mSources.IsEmpty()) {
           LOG(("Audio device %d reallocated", mCapIndex));
         } else {
           LOG(("Audio device %d allocated shared", mCapIndex));
@@ -367,56 +495,32 @@ MediaEngineWebRTCMicrophoneSource::Updat
       break;
 
     default:
       LOG(("Audio device %d in ignored state %d", mCapIndex, mState));
       break;
   }
 
   if (sChannelsOpen > 0) {
-    int error;
+    UpdateAGCSettingsIfNeeded(prefs.mAgcOn, static_cast<AgcModes>(prefs.mAgc));
+    UpdateNSSettingsIfNeeded(prefs.mNoiseOn, static_cast<NsModes>(prefs.mNoise));
+    UpdateAECSettingsIfNeeded(prefs.mAecOn, static_cast<EcModes>(prefs.mAec));
 
-    error = mVoEProcessing->SetEcStatus(prefs.mAecOn, (webrtc::EcModes)prefs.mAec);
-    if (error) {
-      LOG(("%s Error setting Echo Status: %d ",__FUNCTION__, error));
-      // Overhead of capturing all the time is very low (<0.1% of an audio only call)
-      if (prefs.mAecOn) {
-        error = mVoEProcessing->SetEcMetricsStatus(true);
-        if (error) {
-          LOG(("%s Error setting Echo Metrics: %d ",__FUNCTION__, error));
-        }
-      }
-    }
-    error = mVoEProcessing->SetAgcStatus(prefs.mAgcOn, (webrtc::AgcModes)prefs.mAgc);
-    if (error) {
-      LOG(("%s Error setting AGC Status: %d ",__FUNCTION__, error));
-    }
-    error = mVoEProcessing->SetNsStatus(prefs.mNoiseOn, (webrtc::NsModes)prefs.mNoise);
-    if (error) {
-      LOG(("%s Error setting NoiseSuppression Status: %d ",__FUNCTION__, error));
-    }
+    webrtc::Config config;
+    config.Set<webrtc::ExtendedFilter>(new webrtc::ExtendedFilter(mExtendedFilter));
+    config.Set<webrtc::DelayAgnostic>(new webrtc::DelayAgnostic(mDelayAgnostic));
+    mAudioProcessing->SetExtraOptions(config);
   }
 
-  // we don't allow switching from non-fast-path to fast-path on the fly yet
-  if (mState != kStarted) {
-    mSkipProcessing = !(prefs.mAecOn || prefs.mAgcOn || prefs.mNoiseOn);
-    if (mSkipProcessing) {
-      mSampleFrequency = MediaEngine::USE_GRAPH_RATE;
-    } else {
-      // make sure we route a copy of the mixed audio output of this MSG to the
-      // AEC
-      if (!mAudioOutputObserver) {
-        mAudioOutputObserver = new AudioOutputObserver();
-      }
-    }
-  }
   SetLastPrefs(prefs);
   return NS_OK;
 }
 
+#undef HANDLE_APM_ERROR
+
 void
 MediaEngineWebRTCMicrophoneSource::SetLastPrefs(
     const MediaEnginePrefs& aPrefs)
 {
   mLastPrefs = aPrefs;
 
   RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
 
@@ -574,23 +678,35 @@ MediaEngineWebRTCMicrophoneSource::Notif
 
 void
 MediaEngineWebRTCMicrophoneSource::PacketizeAndProcess(MediaStreamGraph* aGraph,
                                                        const AudioDataValue* aBuffer,
                                                        size_t aFrames,
                                                        TrackRate aRate,
                                                        uint32_t aChannels)
 {
-  // This will call Process() with data coming out of the AEC/NS/AGC/etc chain
+  MOZ_ASSERT(!PassThrough(), "This should be bypassed when in PassThrough mode.");
+  size_t offset = 0;
+
   if (!mPacketizer ||
       mPacketizer->PacketSize() != aRate/100u ||
       mPacketizer->Channels() != aChannels) {
     // It's ok to drop the audio still in the packetizer here.
     mPacketizer =
-      new AudioPacketizer<AudioDataValue, int16_t>(aRate/100, aChannels);
+      new AudioPacketizer<AudioDataValue, AudioDataValue>(aRate/100, aChannels);
+  }
+
+  // On initial capture, throw away all far-end data except the most recent sample
+  // since it's already irrelevant and we want to keep avoid confusing the AEC far-end
+  // input code with "old" audio.
+  if (!mStarted) {
+    mStarted  = true;
+    while (mAudioOutputObserver->Size() > 1) {
+      free(mAudioOutputObserver->Pop()); // only call if size() > 0
+    }
   }
 
   mPacketizer->Input(aBuffer, static_cast<uint32_t>(aFrames));
 
   while (mPacketizer->PacketsAvailable()) {
     uint32_t samplesPerPacket = mPacketizer->PacketSize() *
       mPacketizer->Channels();
     if (mInputBuffer.Length() < samplesPerPacket) {
@@ -699,48 +815,41 @@ MediaEngineWebRTCMicrophoneSource::Notif
     InsertInGraph<AudioDataValue>(aBuffer, aFrames, aChannels);
   } else {
     PacketizeAndProcess(aGraph, aBuffer, aFrames, aRate, aChannels);
   }
 }
 
 #define ResetProcessingIfNeeded(_processing)                        \
 do {                                                                \
-  webrtc::_processing##Modes mode;                                  \
-  int rv = mVoEProcessing->Get##_processing##Status(enabled, mode); \
-  if (rv) {                                                         \
-    NS_WARNING("Could not get the status of the "                   \
-     #_processing " on device change.");                            \
-    return;                                                         \
-  }                                                                 \
+  bool enabled = mAudioProcessing->_processing()->is_enabled();     \
                                                                     \
   if (enabled) {                                                    \
-    rv = mVoEProcessing->Set##_processing##Status(!enabled);        \
+    int rv = mAudioProcessing->_processing()->Enable(!enabled);     \
+    if (rv) {                                                       \
+      NS_WARNING("Could not reset the status of the "               \
+      #_processing " on device change.");                           \
+      return;                                                       \
+    }                                                               \
+    rv = mAudioProcessing->_processing()->Enable(enabled);          \
     if (rv) {                                                       \
       NS_WARNING("Could not reset the status of the "               \
       #_processing " on device change.");                           \
       return;                                                       \
     }                                                               \
                                                                     \
-    rv = mVoEProcessing->Set##_processing##Status(enabled);         \
-    if (rv) {                                                       \
-      NS_WARNING("Could not reset the status of the "               \
-      #_processing " on device change.");                           \
-      return;                                                       \
-    }                                                               \
   }                                                                 \
 }  while(0)
 
 void
 MediaEngineWebRTCMicrophoneSource::DeviceChanged() {
   // Reset some processing
-  bool enabled;
-  ResetProcessingIfNeeded(Agc);
-  ResetProcessingIfNeeded(Ec);
-  ResetProcessingIfNeeded(Ns);
+  ResetProcessingIfNeeded(gain_control);
+  ResetProcessingIfNeeded(echo_cancellation);
+  ResetProcessingIfNeeded(noise_suppression);
 }
 
 // mState records if a channel is allocated (slightly redundantly to mChannel)
 void
 MediaEngineWebRTCMicrophoneSource::FreeChannel()
 {
   if (mState != kReleased) {
     mState = kReleased;
@@ -791,18 +900,16 @@ MediaEngineWebRTCMicrophoneSource::Shutd
   }
 
   while (mRegisteredHandles.Length()) {
     MOZ_ASSERT(mState == kAllocated || mState == kStopped);
     // on last Deallocate(), FreeChannel()s and DeInit()s if all channels are released
     Deallocate(mRegisteredHandles[0].get());
   }
   MOZ_ASSERT(mState == kReleased);
-
-  mAudioInput = nullptr;
 }
 
 void
 MediaEngineWebRTCAudioCaptureSource::GetName(nsAString &aName) const
 {
   aName.AssignLiteral("AudioCapture");
 }
 
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -491,31 +491,31 @@ pref("media.peerconnection.use_document_
 pref("media.peerconnection.identity.enabled", true);
 pref("media.peerconnection.identity.timeout", 10000);
 pref("media.peerconnection.ice.stun_client_maximum_transmits", 7);
 pref("media.peerconnection.ice.trickle_grace_period", 5000);
 pref("media.peerconnection.ice.no_host", false);
 pref("media.peerconnection.ice.default_address_only", false);
 pref("media.peerconnection.ice.proxy_only", false);
 
-// These values (aec, agc, and noice) are from media/webrtc/trunk/webrtc/common_types.h
+// These values (aec, agc, and noise) are from media/webrtc/trunk/webrtc/common_types.h
 // kXxxUnchanged = 0, kXxxDefault = 1, and higher values are specific to each
 // setting (for Xxx = Ec, Agc, or Ns).  Defaults are all set to kXxxDefault here.
 pref("media.peerconnection.turn.disable", false);
 #if defined(MOZ_WEBRTC_HARDWARE_AEC_NS)
 pref("media.getusermedia.aec_enabled", false);
 pref("media.getusermedia.noise_enabled", false);
 #else
 pref("media.getusermedia.aec_enabled", true);
 pref("media.getusermedia.noise_enabled", true);
 #endif
 pref("media.getusermedia.aec_extended_filter", true);
 pref("media.getusermedia.noise", 1);
 pref("media.getusermedia.agc_enabled", false);
-pref("media.getusermedia.agc", 1);
+pref("media.getusermedia.agc", 3); // kAgcAdaptiveDigital
 // capture_delay: Adjustments for OS-specific input delay (lower bound)
 // playout_delay: Adjustments for OS-specific AudioStream+cubeb+output delay (lower bound)
 // full_duplex: enable cubeb full-duplex capture/playback
 #if defined(XP_MACOSX)
 pref("media.peerconnection.capture_delay", 50);
 pref("media.navigator.audio.full_duplex", true);
 #elif defined(XP_WIN)
 pref("media.peerconnection.capture_delay", 50);