Bug 1628779 - Pass the audio rountrip latency to the echo canceller, and disable extended filter and delay agnostic AEC, on macOS Nightly. r=achronop
authorPaul Adenot <paul@paul.cx>
Thu, 28 May 2020 09:52:52 +0000
changeset 532743 cc82aacadf8040bc92ab2fd97ec8b6d03bac1eb7
parent 532742 5e0b79c5f7e1e8274a632fbbedcef32ff42cdda1
child 532744 016ee047a1917b3d878cc6d8d322c1ca8e53b12b
push id117348
push userpadenot@mozilla.com
push dateThu, 28 May 2020 09:58:56 +0000
treeherderautoland@5e969a4bab42 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersachronop
bugs1628779
milestone78.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1628779 - Pass the audio rountrip latency to the echo canceller, and disable extended filter and delay agnostic AEC, on macOS Nightly. r=achronop Differential Revision: https://phabricator.services.mozilla.com/D75335
dom/media/MediaManager.cpp
dom/media/webrtc/MediaEnginePrefs.h
dom/media/webrtc/MediaEngineWebRTCAudio.cpp
dom/media/webrtc/MediaEngineWebRTCAudio.h
modules/libpref/init/all.js
--- a/dom/media/MediaManager.cpp
+++ b/dom/media/MediaManager.cpp
@@ -1906,16 +1906,17 @@ MediaManager::MediaManager(UniquePtr<bas
   mPrefs.mFreq = 1000;  // 1KHz test tone
   mPrefs.mWidth = 0;    // adaptive default
   mPrefs.mHeight = 0;   // adaptive default
   mPrefs.mFPS = MediaEnginePrefs::DEFAULT_VIDEO_FPS;
   mPrefs.mAecOn = false;
   mPrefs.mUseAecMobile = false;
   mPrefs.mAgcOn = false;
   mPrefs.mHPFOn = false;
+  mPrefs.mExperimentalInputProcessing = false;
   mPrefs.mNoiseOn = false;
   mPrefs.mExtendedFilter = true;
   mPrefs.mDelayAgnostic = true;
   mPrefs.mFakeDeviceChangeEventOn = false;
 #ifdef MOZ_WEBRTC
   mPrefs.mAec =
       webrtc::EchoCancellation::SuppressionLevel::kModerateSuppression;
   mPrefs.mAgc = webrtc::GainControl::Mode::kAdaptiveDigital;
@@ -1933,24 +1934,27 @@ MediaManager::MediaManager(UniquePtr<bas
       do_GetService("@mozilla.org/preferences-service;1", &rv);
   if (NS_SUCCEEDED(rv)) {
     nsCOMPtr<nsIPrefBranch> branch = do_QueryInterface(prefs);
     if (branch) {
       GetPrefs(branch, nullptr);
     }
   }
   LOG("%s: default prefs: %dx%d @%dfps, %dHz test tones, aec: %s,"
-      "agc: %s, hpf: %s, noise: %s, aec level: %d, agc level: %d, noise level: "
+      "agc: %s, hpf: %s, experimental_input_processing: %s, noise: %s, aec "
+      "level: %d, agc level: %d, noise level: "
       "%d, aec mobile routing mode: %d,"
       "extended aec %s, delay_agnostic %s "
       "channels %d",
       __FUNCTION__, mPrefs.mWidth, mPrefs.mHeight, mPrefs.mFPS, mPrefs.mFreq,
       mPrefs.mAecOn ? "on" : "off", mPrefs.mAgcOn ? "on" : "off",
-      mPrefs.mHPFOn ? "on" : "off", mPrefs.mNoiseOn ? "on" : "off", mPrefs.mAec,
-      mPrefs.mAgc, mPrefs.mNoise, mPrefs.mRoutingMode,
+      mPrefs.mHPFOn ? "on" : "off",
+      mPrefs.mExperimentalInputProcessing ? "on" : "off",
+      mPrefs.mNoiseOn ? "on" : "off", mPrefs.mAec, mPrefs.mAgc, mPrefs.mNoise,
+      mPrefs.mRoutingMode,
       mPrefs.mExtendedFilter ? "on" : "off",
       mPrefs.mDelayAgnostic ? "on" : "off", mPrefs.mChannels);
 }
 
 NS_IMPL_ISUPPORTS(MediaManager, nsIMediaManagerService, nsIObserver)
 
 /* static */
 StaticRefPtr<MediaManager> MediaManager::sSingleton;
@@ -2051,16 +2055,18 @@ MediaManager* MediaManager::Get() {
       prefs->AddObserver("media.getusermedia.agc_enabled", sSingleton, false);
       prefs->AddObserver("media.getusermedia.agc", sSingleton, false);
       prefs->AddObserver("media.getusermedia.hpf_enabled", sSingleton, false);
       prefs->AddObserver("media.getusermedia.noise_enabled", sSingleton, false);
       prefs->AddObserver("media.getusermedia.noise", sSingleton, false);
       prefs->AddObserver("media.ondevicechange.fakeDeviceChangeEvent.enabled",
                          sSingleton, false);
       prefs->AddObserver("media.getusermedia.channels", sSingleton, false);
+      prefs->AddObserver("media.getusermedia.experimental_input_processing",
+                         sSingleton, false);
 #endif
     }
 
     // Prepare async shutdown
 
     class Blocker : public media::ShutdownBlocker {
      public:
       Blocker()
@@ -3494,16 +3500,18 @@ void MediaManager::GetPrefs(nsIPrefBranc
           &mPrefs.mHeight);
   GetPref(aBranch, "media.navigator.video.default_fps", aData, &mPrefs.mFPS);
   GetPref(aBranch, "media.navigator.audio.fake_frequency", aData,
           &mPrefs.mFreq);
 #ifdef MOZ_WEBRTC
   GetPrefBool(aBranch, "media.getusermedia.aec_enabled", aData, &mPrefs.mAecOn);
   GetPrefBool(aBranch, "media.getusermedia.agc_enabled", aData, &mPrefs.mAgcOn);
   GetPrefBool(aBranch, "media.getusermedia.hpf_enabled", aData, &mPrefs.mHPFOn);
+  GetPrefBool(aBranch, "media.getusermedia.experimental_input_processing",
+              aData, &mPrefs.mExperimentalInputProcessing);
   GetPrefBool(aBranch, "media.getusermedia.noise_enabled", aData,
               &mPrefs.mNoiseOn);
   GetPref(aBranch, "media.getusermedia.aec", aData, &mPrefs.mAec);
   GetPref(aBranch, "media.getusermedia.agc", aData, &mPrefs.mAgc);
   GetPref(aBranch, "media.getusermedia.noise", aData, &mPrefs.mNoise);
   GetPref(aBranch, "media.getusermedia.aecm_output_routing", aData,
           &mPrefs.mRoutingMode);
   GetPrefBool(aBranch, "media.getusermedia.aec_extended_filter", aData,
@@ -3551,16 +3559,18 @@ void MediaManager::Shutdown() {
     prefs->RemoveObserver("media.navigator.video.default_height", this);
     prefs->RemoveObserver("media.navigator.video.default_fps", this);
     prefs->RemoveObserver("media.navigator.audio.fake_frequency", this);
 #ifdef MOZ_WEBRTC
     prefs->RemoveObserver("media.getusermedia.aec_enabled", this);
     prefs->RemoveObserver("media.getusermedia.aec", this);
     prefs->RemoveObserver("media.getusermedia.agc_enabled", this);
     prefs->RemoveObserver("media.getusermedia.hpf_enabled", this);
+    prefs->RemoveObserver("media.getusermedia.experimental_input_processing",
+                          this);
     prefs->RemoveObserver("media.getusermedia.agc", this);
     prefs->RemoveObserver("media.getusermedia.noise_enabled", this);
     prefs->RemoveObserver("media.getusermedia.noise", this);
     prefs->RemoveObserver("media.ondevicechange.fakeDeviceChangeEvent.enabled",
                           this);
     prefs->RemoveObserver("media.getusermedia.channels", this);
 #endif
   }
--- a/dom/media/webrtc/MediaEnginePrefs.h
+++ b/dom/media/webrtc/MediaEnginePrefs.h
@@ -24,16 +24,17 @@ class MediaEnginePrefs {
       : mWidth(0),
         mHeight(0),
         mFPS(0),
         mFreq(0),
         mAecOn(false),
         mUseAecMobile(false),
         mAgcOn(false),
         mHPFOn(false),
+        mExperimentalInputProcessing(false),
         mNoiseOn(false),
         mAec(0),
         mAgc(0),
         mRoutingMode(0),
         mNoise(0),
         mExtendedFilter(false),
         mDelayAgnostic(false),
         mFakeDeviceChangeEventOn(false),
@@ -42,16 +43,17 @@ class MediaEnginePrefs {
   int32_t mWidth;
   int32_t mHeight;
   int32_t mFPS;
   int32_t mFreq;  // for test tones (fake:true)
   bool mAecOn;
   bool mUseAecMobile;
   bool mAgcOn;
   bool mHPFOn;
+  bool mExperimentalInputProcessing;
   bool mNoiseOn;
   int32_t mAec;
   int32_t mAgc;
   int32_t mRoutingMode;
   int32_t mNoise;
   bool mExtendedFilter;
   bool mDelayAgnostic;
   bool mFakeDeviceChangeEventOn;
--- a/dom/media/webrtc/MediaEngineWebRTCAudio.cpp
+++ b/dom/media/webrtc/MediaEngineWebRTCAudio.cpp
@@ -160,55 +160,60 @@ nsresult MediaEngineWebRTCMicrophoneSour
 
   mCurrentPrefs = outputPrefs;
 
   return NS_OK;
 }
 
 void MediaEngineWebRTCMicrophoneSource::UpdateAECSettings(
     bool aEnable, bool aUseAecMobile, EchoCancellation::SuppressionLevel aLevel,
-    EchoControlMobile::RoutingMode aRoutingMode) {
+    EchoControlMobile::RoutingMode aRoutingMode,
+    bool aExperimentalInputProcessing) {
   AssertIsOnOwningThread();
 
   RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
   NS_DispatchToMainThread(NS_NewRunnableFunction(
-      __func__,
-      [that, track = mTrack, aEnable, aUseAecMobile, aLevel, aRoutingMode] {
+      __func__, [that, track = mTrack, aEnable, aUseAecMobile, aLevel,
+                 aRoutingMode, aExperimentalInputProcessing] {
         class Message : public ControlMessage {
          public:
           Message(AudioInputProcessing* aInputProcessing, bool aEnable,
                   bool aUseAecMobile, EchoCancellation::SuppressionLevel aLevel,
-                  EchoControlMobile::RoutingMode aRoutingMode)
+                  EchoControlMobile::RoutingMode aRoutingMode,
+                  bool aExperimentalInputProcessing)
               : ControlMessage(nullptr),
                 mInputProcessing(aInputProcessing),
                 mEnable(aEnable),
                 mUseAecMobile(aUseAecMobile),
                 mLevel(aLevel),
-                mRoutingMode(aRoutingMode) {}
+                mRoutingMode(aRoutingMode),
+                mExperimentalInputProcessing(aExperimentalInputProcessing) {}
 
           void Run() override {
             mInputProcessing->UpdateAECSettings(mEnable, mUseAecMobile, mLevel,
-                                                mRoutingMode);
+                                                mRoutingMode,
+                                                mExperimentalInputProcessing);
           }
 
          protected:
           RefPtr<AudioInputProcessing> mInputProcessing;
           bool mEnable;
           bool mUseAecMobile;
           EchoCancellation::SuppressionLevel mLevel;
           EchoControlMobile::RoutingMode mRoutingMode;
+          bool mExperimentalInputProcessing;
         };
 
         if (track->IsDestroyed()) {
           return;
         }
 
-        track->GraphImpl()->AppendMessage(
-            MakeUnique<Message>(that->mInputProcessing, aEnable, aUseAecMobile,
-                                aLevel, aRoutingMode));
+        track->GraphImpl()->AppendMessage(MakeUnique<Message>(
+            that->mInputProcessing, aEnable, aUseAecMobile, aLevel,
+            aRoutingMode, aExperimentalInputProcessing));
       }));
 }
 
 void MediaEngineWebRTCMicrophoneSource::UpdateAGCSettings(
     bool aEnable, GainControl::Mode aMode) {
   AssertIsOnOwningThread();
 
   RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
@@ -355,20 +360,25 @@ void MediaEngineWebRTCMicrophoneSource::
                       static_cast<webrtc::GainControl::Mode>(aPrefs.mAgc));
     UpdateNSSettings(
         aPrefs.mNoiseOn,
         static_cast<webrtc::NoiseSuppression::Level>(aPrefs.mNoise));
     UpdateAECSettings(
         aPrefs.mAecOn, aPrefs.mUseAecMobile,
         static_cast<webrtc::EchoCancellation::SuppressionLevel>(aPrefs.mAec),
         static_cast<webrtc::EchoControlMobile::RoutingMode>(
-            aPrefs.mRoutingMode));
+            aPrefs.mRoutingMode),
+        aPrefs.mExperimentalInputProcessing);
     UpdateHPFSettings(aPrefs.mHPFOn);
 
-    UpdateAPMExtraOptions(mExtendedFilter, mDelayAgnostic);
+    if (!aPrefs.mExperimentalInputProcessing) {
+      UpdateAPMExtraOptions(mExtendedFilter, mDelayAgnostic);
+    } else {
+      LOG("Using experimental input processing");
+    }
   }
 
   RefPtr<MediaEngineWebRTCMicrophoneSource> that = this;
   NS_DispatchToMainThread(
       NS_NewRunnableFunction(__func__, [that, track = mTrack, prefs = aPrefs] {
         that->mSettings->mEchoCancellation.Value() = prefs.mAecOn;
         that->mSettings->mAutoGainControl.Value() = prefs.mAgcOn;
         that->mSettings->mNoiseSuppression.Value() = prefs.mNoiseOn;
@@ -627,17 +637,18 @@ AudioInputProcessing::AudioInputProcessi
       ,
       mLastCallbackAppendTime(0)
 #endif
       ,
       mLiveFramesAppended(false),
       mLiveSilenceAppended(false),
       mPrincipal(aPrincipalHandle),
       mEnabled(false),
-      mEnded(false) {
+      mEnded(false),
+      mExperimentalInputProcessing(false) {
 }
 
 void AudioInputProcessing::Disconnect(MediaTrackGraphImpl* aGraph) {
   // This method is just for asserts.
   MOZ_ASSERT(aGraph->OnGraphThread());
 }
 
 void MediaEngineWebRTCMicrophoneSource::Shutdown() {
@@ -681,17 +692,18 @@ void AudioInputProcessing::SetRequestedI
     if (rv != AudioProcessing::kNoError) {         \
       MOZ_ASSERT_UNREACHABLE("APM error in " #fn); \
       return;                                      \
     }                                              \
   } while (0);
 
 void AudioInputProcessing::UpdateAECSettings(
     bool aEnable, bool aUseAecMobile, EchoCancellation::SuppressionLevel aLevel,
-    EchoControlMobile::RoutingMode aRoutingMode) {
+    EchoControlMobile::RoutingMode aRoutingMode,
+    bool aExperimentalInputProcessing) {
   if (aUseAecMobile) {
     HANDLE_APM_ERROR(mAudioProcessing->echo_control_mobile()->Enable(aEnable));
     HANDLE_APM_ERROR(mAudioProcessing->echo_control_mobile()->set_routing_mode(
         aRoutingMode));
     HANDLE_APM_ERROR(mAudioProcessing->echo_cancellation()->Enable(false));
   } else {
     if (aLevel != EchoCancellation::SuppressionLevel::kLowSuppression &&
         aLevel != EchoCancellation::SuppressionLevel::kModerateSuppression &&
@@ -702,16 +714,18 @@ void AudioInputProcessing::UpdateAECSett
       aLevel = EchoCancellation::SuppressionLevel::kModerateSuppression;
     }
 
     HANDLE_APM_ERROR(mAudioProcessing->echo_control_mobile()->Enable(false));
     HANDLE_APM_ERROR(mAudioProcessing->echo_cancellation()->Enable(aEnable));
     HANDLE_APM_ERROR(
         mAudioProcessing->echo_cancellation()->set_suppression_level(aLevel));
   }
+
+  mExperimentalInputProcessing = aExperimentalInputProcessing;
 }
 
 void AudioInputProcessing::UpdateAGCSettings(bool aEnable,
                                              GainControl::Mode aMode) {
   if (aMode != GainControl::Mode::kAdaptiveAnalog &&
       aMode != GainControl::Mode::kAdaptiveDigital &&
       aMode != GainControl::Mode::kFixedDigital) {
     LOG_ERROR("Attempt to set invalid AGC mode %d", static_cast<int>(aMode));
@@ -969,18 +983,27 @@ void AudioInputProcessing::PacketizeAndP
     // Deinterleave to mInputBuffer, pointed to by inputBufferChannelPointers.
     Deinterleave(packet, mPacketizerInput->PacketSize(), aChannels,
                  deinterleavedPacketizedInputDataChannelPointers.Elements());
 
     StreamConfig inputConfig(aRate, aChannels,
                              false /* we don't use typing detection*/);
     StreamConfig outputConfig = inputConfig;
 
-    // Bug 1404965: Get the right delay here, it saves some work down the line.
-    mAudioProcessing->set_stream_delay_ms(0);
+    if (mExperimentalInputProcessing) {
+      MediaTrackGraphImpl* graphImpl = mTrack->GraphImpl();
+      double roundtripLatencyMS = (graphImpl->AudioInputLatencyGraphThread() +
+                                   graphImpl->AudioOutputLatencyGraphThread()) *
+                                  1000;
+      LOG_FRAME("Audio roundtrip latency %lfms", roundtripLatencyMS);
+      mAudioProcessing->set_stream_delay_ms(
+          static_cast<uint32_t>(roundtripLatencyMS));
+    } else {
+      mAudioProcessing->set_stream_delay_ms(0);
+    }
 
     // Bug 1414837: find a way to not allocate here.
     CheckedInt<size_t> bufferSize(sizeof(float));
     bufferSize *= mPacketizerInput->PacketSize();
     bufferSize *= aChannels;
     RefPtr<SharedBuffer> buffer = SharedBuffer::Create(bufferSize);
 
     // Prepare channel pointers to the SharedBuffer created above.
@@ -1141,17 +1164,17 @@ void AudioInputProcessing::DeviceChanged
 
 void AudioInputProcessing::End() { mEnded = true; }
 
 nsString MediaEngineWebRTCAudioCaptureSource::GetName() const {
   return NS_LITERAL_STRING(u"AudioCapture");
 }
 
 nsCString MediaEngineWebRTCAudioCaptureSource::GetUUID() const {
-  nsID uuid;
+  nsID uuid = nsID();
   char uuidBuffer[NSID_LENGTH];
   nsCString asciiString;
   ErrorResult rv;
 
   rv = nsContentUtils::GenerateUUIDInPlace(uuid);
   if (rv.Failed()) {
     return NS_LITERAL_CSTRING("");
   }
--- a/dom/media/webrtc/MediaEngineWebRTCAudio.h
+++ b/dom/media/webrtc/MediaEngineWebRTCAudio.h
@@ -89,17 +89,18 @@ class MediaEngineWebRTCMicrophoneSource 
    */
   void ApplySettings(const MediaEnginePrefs& aPrefs);
 
   /**
    * Sent the AudioProcessingModule parameter for a given processing algorithm.
    */
   void UpdateAECSettings(bool aEnable, bool aUseAecMobile,
                          webrtc::EchoCancellation::SuppressionLevel aLevel,
-                         webrtc::EchoControlMobile::RoutingMode aRoutingMode);
+                         webrtc::EchoControlMobile::RoutingMode aRoutingMode,
+                         bool aExperimentalInputProcessing);
   void UpdateAGCSettings(bool aEnable, webrtc::GainControl::Mode aMode);
   void UpdateHPFSettings(bool aEnable);
   void UpdateNSSettings(bool aEnable, webrtc::NoiseSuppression::Level aLevel);
   void UpdateAPMExtraOptions(bool aExtendedFilter, bool aDelayAgnostic);
 
   PrincipalHandle mPrincipal = PRINCIPAL_HANDLE_NONE;
 
   const RefPtr<AudioDeviceInfo> mDeviceInfo;
@@ -187,17 +188,18 @@ class AudioInputProcessing : public Audi
   // This is true when all processing is disabled, we can skip
   // packetization, resampling and other processing passes.
   bool PassThrough(MediaTrackGraphImpl* aGraphImpl) const;
 
   // This allow changing the APM options, enabling or disabling processing
   // steps.
   void UpdateAECSettings(bool aEnable, bool aUseAecMobile,
                          webrtc::EchoCancellation::SuppressionLevel aLevel,
-                         webrtc::EchoControlMobile::RoutingMode aRoutingMode);
+                         webrtc::EchoControlMobile::RoutingMode aRoutingMode,
+                         bool aExperimentalInputProcessing);
   void UpdateAGCSettings(bool aEnable, webrtc::GainControl::Mode aMode);
   void UpdateHPFSettings(bool aEnable);
   void UpdateNSSettings(bool aEnable, webrtc::NoiseSuppression::Level aLevel);
   void UpdateAPMExtraOptions(bool aExtendedFilter, bool aDelayAgnostic);
 
   void End();
 
  private:
@@ -246,16 +248,19 @@ class AudioInputProcessing : public Audi
   // Principal for the data that flows through this class.
   const PrincipalHandle mPrincipal;
   // Whether or not this MediaEngine is enabled. If it's not enabled, it
   // operates in "pull" mode, and we append silence only, releasing the audio
   // input track.
   bool mEnabled;
   // Whether or not we've ended and removed the track in the SourceMediaTrack
   bool mEnded;
+  // Flag that diables extended filter and delay-agnostic AEC, and passes
+  // roundtrip time to the AEC.
+  bool mExperimentalInputProcessing;
 };
 
 // This class is created on the media thread, as part of Start(), then entirely
 // self-sustained until destruction, just forwarding calls to Pull().
 class AudioInputProcessingPullListener : public MediaTrackListener {
  public:
   explicit AudioInputProcessingPullListener(
       RefPtr<AudioInputProcessing> aInputProcessing)
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -521,16 +521,21 @@ pref("media.videocontrols.picture-in-pic
   pref("media.getusermedia.use_aec_mobile", false);
   pref("media.getusermedia.aec", 1); // kModerateSuppression
   pref("media.getusermedia.aec_extended_filter", true);
   pref("media.getusermedia.noise", 1); // kModerate
   pref("media.getusermedia.agc_enabled", true);
   pref("media.getusermedia.agc", 1); // kAdaptiveDigital
   pref("media.getusermedia.hpf_enabled", true);
   pref("media.getusermedia.aecm_output_routing", 3); // kSpeakerphone
+#if defined(NIGHTLY_BUILD) && defined(XP_MACOSX)
+  pref("media.getusermedia.experimental_input_processing", true);
+#else
+  pref("media.getusermedia.experimental_input_processing", false);
+#endif
 #endif // MOZ_WEBRTC
 
 #if !defined(ANDROID)
   pref("media.getusermedia.screensharing.enabled", true);
 #endif
 
 pref("media.getusermedia.audiocapture.enabled", false);