Bug 1580659 - part5 : stop audio capturing explicitly. r=chunmin,karlt
authorAlastor Wu <alwu@mozilla.com>
Tue, 24 Sep 2019 06:55:39 +0000
changeset 494862 c4bc95efde9a5741833abe92febd7ad1875cc070
parent 494861 46e39dfd51d2a7c66d267e4a88f0549cbe4363cc
child 494863 12f46e16c72c7be3d8bfd428c3937a9848e662f4
push id114131
push userdluca@mozilla.com
push dateThu, 26 Sep 2019 09:47:34 +0000
treeherdermozilla-inbound@1dc1a755079a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschunmin, karlt
bugs1580659
milestone71.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1580659 - part5 : stop audio capturing explicitly. r=chunmin,karlt As we start audio capturing explicitly, we should also take responsibility to stop audio capturing when we don't need it. We were hiding too many details on `AudioChannelAgent` before, which allow us hard to know who and where we handle audio capturing. Differential Revision: https://phabricator.services.mozilla.com/D45752
dom/audiochannel/AudioChannelAgent.cpp
dom/audiochannel/AudioChannelAgent.h
dom/audiochannel/AudioChannelService.cpp
dom/audiochannel/AudioChannelService.h
dom/html/HTMLMediaElement.cpp
dom/media/webaudio/AudioDestinationNode.cpp
dom/media/webaudio/AudioDestinationNode.h
--- a/dom/audiochannel/AudioChannelAgent.cpp
+++ b/dom/audiochannel/AudioChannelAgent.cpp
@@ -253,17 +253,17 @@ void AudioChannelAgent::WindowSuspendCha
   MOZ_LOG(AudioChannelService::GetAudioChannelLog(), LogLevel::Debug,
           ("AudioChannelAgent, WindowSuspendChanged, this = %p, "
            "suspended = %s\n",
            this, SuspendTypeToStr(aSuspend)));
 
   callback->WindowSuspendChanged(aSuspend);
 }
 
-AudioPlaybackConfig AudioChannelAgent::GetMediaConfig() {
+AudioPlaybackConfig AudioChannelAgent::GetMediaConfig() const {
   RefPtr<AudioChannelService> service = AudioChannelService::GetOrCreate();
   AudioPlaybackConfig config(1.0, false, nsISuspendedTypes::NONE_SUSPENDED);
   if (service) {
     config = service->GetMediaConfig(mWindow);
   }
   return config;
 }
 
@@ -292,15 +292,19 @@ void AudioChannelAgent::WindowAudioCaptu
   MOZ_LOG(AudioChannelService::GetAudioChannelLog(), LogLevel::Debug,
           ("AudioChannelAgent, WindowAudioCaptureChanged, this = %p, "
            "capture = %d\n",
            this, aCapture));
 
   callback->WindowAudioCaptureChanged(aCapture);
 }
 
+bool AudioChannelAgent::IsWindowAudioCapturingEnabled() const {
+  return GetMediaConfig().mCapturedAudio;
+}
+
 bool AudioChannelAgent::IsPlayingStarted() const { return mIsRegToService; }
 
 bool AudioChannelAgent::ShouldBlockMedia() const {
   return mWindow
              ? mWindow->GetMediaSuspend() == nsISuspendedTypes::SUSPENDED_BLOCK
              : false;
 }
--- a/dom/audiochannel/AudioChannelAgent.h
+++ b/dom/audiochannel/AudioChannelAgent.h
@@ -41,23 +41,24 @@ class AudioChannelAgent : public nsIAudi
   void WindowSuspendChanged(nsSuspendedTypes aSuspend);
   void WindowAudioCaptureChanged(uint64_t aInnerWindowID, bool aCapture);
 
   nsPIDOMWindowOuter* Window() const { return mWindow; }
 
   uint64_t WindowID() const;
   uint64_t InnerWindowID() const;
 
+  bool IsWindowAudioCapturingEnabled() const;
   bool IsPlayingStarted() const;
   bool ShouldBlockMedia() const;
 
  private:
   virtual ~AudioChannelAgent();
 
-  AudioPlaybackConfig GetMediaConfig();
+  AudioPlaybackConfig GetMediaConfig() const;
   bool IsDisposableSuspend(nsSuspendedTypes aSuspend) const;
 
   // Returns mCallback if that's non-null, or otherwise tries to get an
   // nsIAudioChannelAgentCallback out of mWeakCallback.
   already_AddRefed<nsIAudioChannelAgentCallback> GetCallback();
 
   nsresult InitInternal(nsPIDOMWindowInner* aWindow,
                         nsIAudioChannelAgentCallback* aCallback,
--- a/dom/audiochannel/AudioChannelService.cpp
+++ b/dom/audiochannel/AudioChannelService.cpp
@@ -728,17 +728,16 @@ void AudioChannelService::AudioChannelWi
   }
 }
 
 void AudioChannelService::AudioChannelWindow::RemoveAgent(
     AudioChannelAgent* aAgent) {
   MOZ_ASSERT(aAgent);
 
   RemoveAgentAndReduceAgentsNum(aAgent);
-  AudioCapturedChanged(aAgent, AudioCaptureState::eNotCapturing);
   AudioAudibleChanged(aAgent, AudibleState::eNotAudible,
                       AudibleChangedReasons::ePauseStateChanged);
 }
 
 void AudioChannelService::AudioChannelWindow::NotifyMediaBlockStop(
     nsPIDOMWindowOuter* aWindow) {
   if (mShouldSendActiveMediaBlockStopEvent) {
     mShouldSendActiveMediaBlockStopEvent = false;
@@ -784,25 +783,16 @@ void AudioChannelService::AudioChannelWi
   MOZ_ASSERT(mConfig.mNumberOfAgents > 0);
   --mConfig.mNumberOfAgents;
 
   if (mConfig.mNumberOfAgents == 0) {
     NotifyChannelActive(aAgent->WindowID(), false);
   }
 }
 
-void AudioChannelService::AudioChannelWindow::AudioCapturedChanged(
-    AudioChannelAgent* aAgent, AudioCaptureState aCapture) {
-  MOZ_ASSERT(aAgent);
-
-  if (mIsAudioCaptured) {
-    aAgent->WindowAudioCaptureChanged(aAgent->InnerWindowID(), aCapture);
-  }
-}
-
 void AudioChannelService::AudioChannelWindow::AudioAudibleChanged(
     AudioChannelAgent* aAgent, AudibleState aAudible,
     AudibleChangedReasons aReason) {
   MOZ_ASSERT(aAgent);
 
   if (aAudible == AudibleState::eAudible) {
     AppendAudibleAgentIfNotContained(aAgent, aReason);
     NotifyAudioCompetingChanged(aAgent);
--- a/dom/audiochannel/AudioChannelService.h
+++ b/dom/audiochannel/AudioChannelService.h
@@ -174,19 +174,16 @@ class AudioChannelService final : public
     bool mOwningAudioFocus;
 
     // If we've dispatched "activeMediaBlockStart" event, we must dispatch
     // another event "activeMediablockStop" when the window is resumed from
     // suspend-block.
     bool mShouldSendActiveMediaBlockStopEvent;
 
    private:
-    void AudioCapturedChanged(AudioChannelAgent* aAgent,
-                              AudioCaptureState aCapture);
-
     void AppendAudibleAgentIfNotContained(AudioChannelAgent* aAgent,
                                           AudibleChangedReasons aReason);
     void RemoveAudibleAgentIfContained(AudioChannelAgent* aAgent,
                                        AudibleChangedReasons aReason);
 
     void AppendAgentAndIncreaseAgentsNum(AudioChannelAgent* aAgent);
     void RemoveAgentAndReduceAgentsNum(AudioChannelAgent* aAgent);
 
--- a/dom/html/HTMLMediaElement.cpp
+++ b/dom/html/HTMLMediaElement.cpp
@@ -979,17 +979,16 @@ class HTMLMediaElement::AudioChannelAgen
  public:
   NS_DECL_CYCLE_COLLECTING_ISUPPORTS
   NS_DECL_CYCLE_COLLECTION_CLASS(AudioChannelAgentCallback)
 
   explicit AudioChannelAgentCallback(HTMLMediaElement* aOwner)
       : mOwner(aOwner),
         mAudioChannelVolume(1.0),
         mPlayingThroughTheAudioChannel(false),
-        mAudioCapturedByWindow(false),
         mSuspended(nsISuspendedTypes::NONE_SUSPENDED),
         mIsOwnerAudible(IsOwnerAudible()),
         mIsShutDown(false) {
     MOZ_ASSERT(mOwner);
     MaybeCreateAudioChannelAgent();
   }
 
   void UpdateAudioChannelPlayingState(bool aForcePlaying = false) {
@@ -1089,30 +1088,29 @@ class HTMLMediaElement::AudioChannelAgen
                  "this = %p, Error : unknown suspended type!\n",
                  this));
     }
     return NS_OK;
   }
 
   NS_IMETHODIMP WindowAudioCaptureChanged(bool aCapture) override {
     MOZ_ASSERT(mAudioChannelAgent);
-
-    if (mAudioCapturedByWindow != aCapture) {
-      mAudioCapturedByWindow = aCapture;
-      AudioCaptureStreamChangeIfNeeded();
-    }
+    AudioCaptureStreamChangeIfNeeded();
     return NS_OK;
   }
 
   void AudioCaptureStreamChangeIfNeeded() {
     MOZ_ASSERT(!mIsShutDown);
     if (!IsPlayingStarted()) {
       return;
     }
-    mOwner->AudioCaptureStreamChange(mAudioCapturedByWindow);
+
+    MOZ_ASSERT(mAudioChannelAgent);
+    bool isCapturing = mAudioChannelAgent->IsWindowAudioCapturingEnabled();
+    mOwner->AudioCaptureStreamChange(isCapturing);
   }
 
   void NotifyAudioPlaybackChanged(AudibleChangedReasons aReason) {
     MOZ_ASSERT(!mIsShutDown);
     if (!IsPlayingStarted()) {
       return;
     }
 
@@ -1199,16 +1197,19 @@ class HTMLMediaElement::AudioChannelAgen
     mAudioChannelAgent->PullInitialUpdate();
   }
 
   void StopAudioChanelAgent() {
     MOZ_ASSERT(mAudioChannelAgent);
     MOZ_ASSERT(mAudioChannelAgent->IsPlayingStarted());
     mAudioChannelAgent->NotifyStoppedPlaying();
     NotifyMediaStopped(mAudioChannelAgent->WindowID());
+    // If we have started audio capturing before, we have to tell media element
+    // to clear the output capturing stream.
+    mOwner->AudioCaptureStreamChange(false);
   }
 
   void SetSuspended(SuspendTypes aSuspend) {
     if (mSuspended == aSuspend) {
       return;
     }
 
     MaybeNotifyMediaResumed(aSuspend);
@@ -1394,18 +1395,16 @@ class HTMLMediaElement::AudioChannelAgen
 
   RefPtr<AudioChannelAgent> mAudioChannelAgent;
   HTMLMediaElement* mOwner;
 
   // The audio channel volume
   float mAudioChannelVolume;
   // Is this media element playing?
   bool mPlayingThroughTheAudioChannel;
-  // True if the sound is being captured by the window.
-  bool mAudioCapturedByWindow;
   // We have different kinds of suspended cases,
   // - SUSPENDED_PAUSE
   // It's used when we temporary lost platform audio focus. MediaElement can
   // only be resumed when we gain the audio focus again.
   // - SUSPENDED_PAUSE_DISPOSABLE
   // It's used when user press the pause button on the remote media-control.
   // MediaElement can be resumed by remote media-control or via play().
   // - SUSPENDED_BLOCK
--- a/dom/media/webaudio/AudioDestinationNode.cpp
+++ b/dom/media/webaudio/AudioDestinationNode.cpp
@@ -321,17 +321,16 @@ AudioDestinationNode::AudioDestinationNo
                                            bool aAllowedToStart,
                                            uint32_t aNumberOfChannels,
                                            uint32_t aLength)
     : AudioNode(aContext, aNumberOfChannels, ChannelCountMode::Explicit,
                 ChannelInterpretation::Speakers),
       mFramesToProduce(aLength),
       mIsOffline(aIsOffline),
       mAudioChannelSuspended(false),
-      mCaptured(false),
       mAudible(AudioChannelService::AudibleState::eAudible),
       mCreatedTime(TimeStamp::Now()) {
   if (aIsOffline) {
     // The stream is created on demand to avoid creating a graph thread that
     // may not be used.
     return;
   }
 
@@ -391,16 +390,19 @@ AudioNodeStream* AudioDestinationNode::S
 }
 
 void AudioDestinationNode::DestroyAudioChannelAgent() {
   if (mAudioChannelAgent && !Context()->IsOffline()) {
     mAudioChannelAgent->NotifyStoppedPlaying();
     mAudioChannelAgent = nullptr;
     // Reset the state, and it would always be regard as audible.
     mAudible = AudioChannelService::AudibleState::eAudible;
+    if (IsCapturingAudio()) {
+      StopAudioCapturingStream();
+    }
   }
 }
 
 void AudioDestinationNode::DestroyMediaStream() {
   DestroyAudioChannelAgent();
 
   if (!mStream) return;
 
@@ -560,31 +562,46 @@ AudioDestinationNode::WindowAudioCapture
     return NS_OK;
   }
 
   nsCOMPtr<nsPIDOMWindowInner> ownerWindow = GetOwner();
   if (!ownerWindow) {
     return NS_OK;
   }
 
-  if (aCapture != mCaptured) {
-    if (aCapture) {
-      nsCOMPtr<nsPIDOMWindowInner> window = Context()->GetParentObject();
-      uint64_t id = window->WindowID();
-      mCaptureStreamPort =
-          mStream->Graph()->ConnectToCaptureStream(id, mStream);
-    } else {
-      mCaptureStreamPort->Destroy();
-    }
-    mCaptured = aCapture;
+  if (aCapture == IsCapturingAudio()) {
+    return NS_OK;
+  }
+
+  if (aCapture) {
+    StartAudioCapturingStream();
+  } else {
+    StopAudioCapturingStream();
   }
 
   return NS_OK;
 }
 
+bool AudioDestinationNode::IsCapturingAudio() const {
+  return mCaptureStreamPort != nullptr;
+}
+
+void AudioDestinationNode::StartAudioCapturingStream() {
+  MOZ_ASSERT(!IsCapturingAudio());
+  nsCOMPtr<nsPIDOMWindowInner> window = Context()->GetParentObject();
+  uint64_t id = window->WindowID();
+  mCaptureStreamPort = mStream->Graph()->ConnectToCaptureStream(id, mStream);
+}
+
+void AudioDestinationNode::StopAudioCapturingStream() {
+  MOZ_ASSERT(IsCapturingAudio());
+  mCaptureStreamPort->Destroy();
+  mCaptureStreamPort = nullptr;
+}
+
 nsresult AudioDestinationNode::CreateAudioChannelAgent() {
   if (mIsOffline || mAudioChannelAgent) {
     return NS_OK;
   }
 
   mAudioChannelAgent = new AudioChannelAgent();
   nsresult rv = mAudioChannelAgent->InitWithWeakCallback(GetOwner(), this);
   if (NS_WARN_IF(NS_FAILED(rv))) {
@@ -607,16 +624,19 @@ void AudioDestinationNode::NotifyAudible
   AUDIO_CHANNEL_LOG(
       "AudioDestinationNode %p NotifyAudibleStateChanged, audible=%d", this,
       aAudible);
 
   if (!aAudible) {
     mAudioChannelAgent->NotifyStoppedPlaying();
     // Reset the state, and it would always be regard as audible.
     mAudible = AudioChannelService::AudibleState::eAudible;
+    if (IsCapturingAudio()) {
+      StopAudioCapturingStream();
+    }
     return;
   }
 
   if (mDurationBeforeFirstTimeAudible.IsZero()) {
     MOZ_ASSERT(aAudible);
     mDurationBeforeFirstTimeAudible = TimeStamp::Now() - mCreatedTime;
     Telemetry::Accumulate(Telemetry::WEB_AUDIO_BECOMES_AUDIBLE_TIME,
                           mDurationBeforeFirstTimeAudible.ToSeconds());
--- a/dom/media/webaudio/AudioDestinationNode.h
+++ b/dom/media/webaudio/AudioDestinationNode.h
@@ -72,28 +72,35 @@ class AudioDestinationNode final : publi
     MOZ_ASSERT(mIsOffline);
     return mFramesToProduce;
   }
 
  protected:
   virtual ~AudioDestinationNode();
 
  private:
+  // These function are related to audio capturing. We would start capturing
+  // audio if we're starting capturing audio from whole window, and MUST stop
+  // capturing explicitly when we don't need to capture audio any more, because
+  // we have to release the resource we allocated before.
+  bool IsCapturingAudio() const;
+  void StartAudioCapturingStream();
+  void StopAudioCapturingStream();
+
   SelfReference<AudioDestinationNode> mOfflineRenderingRef;
   uint32_t mFramesToProduce;
 
   RefPtr<AudioChannelAgent> mAudioChannelAgent;
   RefPtr<MediaInputPort> mCaptureStreamPort;
 
   RefPtr<Promise> mOfflineRenderingPromise;
 
   bool mIsOffline;
   bool mAudioChannelSuspended;
 
-  bool mCaptured;
   AudioChannelService::AudibleState mAudible;
 
   // These varaibles are used to know how long AudioContext would become audible
   // since it was created.
   TimeStamp mCreatedTime;
   TimeDuration mDurationBeforeFirstTimeAudible;
 };