Bug 1156472 - Part 6 - Connect HTMLMediaElement and AudioContext to the capture stream when capturing is needed. r=roc
--- a/dom/html/HTMLMediaElement.cpp
+++ b/dom/html/HTMLMediaElement.cpp
@@ -2025,16 +2025,17 @@ HTMLMediaElement::HTMLMediaElement(alrea
mAutoplaying(true),
mAutoplayEnabled(true),
mPaused(true),
mMuted(0),
mStatsShowing(false),
mAllowCasting(false),
mIsCasting(false),
mAudioCaptured(false),
+ mAudioCapturedByWindow(false),
mPlayingBeforeSeek(false),
mPlayingThroughTheAudioChannelBeforeSeek(false),
mPausedForInactiveDocumentOrChannel(false),
mEventDeliveryPaused(false),
mWaitingFired(false),
mIsRunningLoadMethod(false),
mIsDoingExplicitLoad(false),
mIsLoadingFromSourceChildren(false),
@@ -2092,16 +2093,21 @@ HTMLMediaElement::~HTMLMediaElement()
}
if (mProgressTimer) {
StopProgress();
}
if (mSrcStream) {
EndSrcMediaStreamPlayback();
}
+ if (mCaptureStreamPort) {
+ mCaptureStreamPort->Destroy();
+ mCaptureStreamPort = nullptr;
+ }
+
NS_ASSERTION(MediaElementTableCount(this, mLoadingSrc) == 0,
"Destroyed media element should no longer be in element table");
if (mChannel) {
mChannel->Cancel(NS_BINDING_ABORTED);
}
WakeLockRelease();
@@ -4470,18 +4476,17 @@ void HTMLMediaElement::UpdateAudioChanne
if (!UseAudioChannelService()) {
return;
}
bool playingThroughTheAudioChannel =
(!mPaused &&
(HasAttr(kNameSpaceID_None, nsGkAtoms::loop) ||
(mReadyState >= nsIDOMHTMLMediaElement::HAVE_CURRENT_DATA &&
- !IsPlaybackEnded() &&
- (!mSrcStream || HasAudio())) ||
+ !IsPlaybackEnded()) ||
mPlayingThroughTheAudioChannelBeforeSeek));
if (playingThroughTheAudioChannel != mPlayingThroughTheAudioChannel) {
mPlayingThroughTheAudioChannel = playingThroughTheAudioChannel;
// If we are not playing, we don't need to create a new audioChannelAgent.
if (!mAudioChannelAgent && !mPlayingThroughTheAudioChannel) {
return;
}
@@ -4499,19 +4504,19 @@ void HTMLMediaElement::UpdateAudioChanne
NotifyAudioChannelAgent(mPlayingThroughTheAudioChannel);
}
}
void
HTMLMediaElement::NotifyAudioChannelAgent(bool aPlaying)
{
- // Immediately check if this should go to the MSG instead of the normal
- // media playback route.
- WindowAudioCaptureChanged();
+ // Immediately check if this should go to the MSG instead of the normal
+ // media playback route.
+ WindowAudioCaptureChanged();
// This is needed to pass nsContentUtils::IsCallerChrome().
// AudioChannel API should not called from content but it can happen that
// this method has some content JS in its stack.
AutoNoJSAPI nojsapi;
if (aPlaying) {
float volume = 0.0;
@@ -4675,21 +4680,43 @@ HTMLMediaElement::GetTopLevelPrincipal()
}
principal = doc->NodePrincipal();
return principal.forget();
}
#endif // MOZ_EME
NS_IMETHODIMP HTMLMediaElement::WindowAudioCaptureChanged()
{
- MOZ_ASSERT(mAudioChannelAgent);
- DebugOnly<bool> captured = OwnerDoc()->GetInnerWindow()->GetAudioCaptured();
-
- // Something is going to happen here!!
- return NS_OK;
+ MOZ_ASSERT(mAudioChannelAgent);
+
+ if (!OwnerDoc()->GetInnerWindow()) {
+ return NS_OK;
+ }
+ bool captured = OwnerDoc()->GetInnerWindow()->GetAudioCaptured();
+
+ if (captured != mAudioCapturedByWindow) {
+ if (captured) {
+ mAudioCapturedByWindow = true;
+ nsCOMPtr<nsPIDOMWindow> window =
+ do_QueryInterface(OwnerDoc()->GetParentObject());
+ uint64_t id = window->WindowID();
+ MediaStreamGraph* msg = MediaStreamGraph::GetInstance();
+
+ if (!mPlaybackStream) {
+ nsRefPtr<DOMMediaStream> stream = CaptureStreamInternal(false, msg);
+ mCaptureStreamPort = msg->ConnectToCaptureStream(id, stream->GetStream());
+ } else {
+ mCaptureStreamPort = msg->ConnectToCaptureStream(id, mPlaybackStream->GetStream());
+ }
+ } else {
+ // TODO: uncapture
+ }
+ }
+
+ return NS_OK;
}
AudioTrackList*
HTMLMediaElement::AudioTracks()
{
if (!mAudioTrackList) {
nsCOMPtr<nsPIDOMWindow> window = do_QueryInterface(OwnerDoc()->GetParentObject());
mAudioTrackList = new AudioTrackList(window, this);
--- a/dom/html/HTMLMediaElement.h
+++ b/dom/html/HTMLMediaElement.h
@@ -1069,16 +1069,19 @@ protected:
// Holds a reference to the DOM wrapper for the MediaStream that we're
// actually playing.
// At most one of mDecoder and mSrcStream can be non-null.
nsRefPtr<DOMMediaStream> mSrcStream;
// Holds a reference to a MediaInputPort connecting mSrcStream to mPlaybackStream.
nsRefPtr<MediaInputPort> mPlaybackStreamInputPort;
+ // Holds a reference to the stream connecting this stream to the capture sink.
+ nsRefPtr<MediaInputPort> mCaptureStreamPort;
+
// Holds a reference to a stream with mSrcStream as input but intended for
// playback. Used so we don't block playback of other video elements
// playing the same mSrcStream.
nsRefPtr<DOMMediaStream> mPlaybackStream;
// Holds references to the DOM wrappers for the MediaStreams that we're
// writing to.
struct OutputMediaStream {
@@ -1278,16 +1281,19 @@ protected:
// True if casting is currently allowed
bool mAllowCasting;
// True if currently casting this video
bool mIsCasting;
// True if the sound is being captured.
bool mAudioCaptured;
+ // True if the sound is being captured by the window.
+ bool mAudioCapturedByWindow;
+
// If TRUE then the media element was actively playing before the currently
// in progress seeking. If FALSE then the media element is either not seeking
// or was not actively playing before the current seek. Used to decide whether
// to raise the 'waiting' event as per 4.7.1.8 in HTML 5 specification.
bool mPlayingBeforeSeek;
// if TRUE then the seek started while content was in active playing state
// if FALSE then the seek started while the content was not playing.
--- a/dom/media/webaudio/AudioDestinationNode.cpp
+++ b/dom/media/webaudio/AudioDestinationNode.cpp
@@ -308,29 +308,27 @@ NS_INTERFACE_MAP_END_INHERITING(AudioNod
NS_IMPL_ADDREF_INHERITED(AudioDestinationNode, AudioNode)
NS_IMPL_RELEASE_INHERITED(AudioDestinationNode, AudioNode)
AudioDestinationNode::AudioDestinationNode(AudioContext* aContext,
bool aIsOffline,
AudioChannel aChannel,
uint32_t aNumberOfChannels,
- uint32_t aLength,
- float aSampleRate)
- : AudioNode(aContext,
- aIsOffline ? aNumberOfChannels : 2,
- ChannelCountMode::Explicit,
- ChannelInterpretation::Speakers)
+ uint32_t aLength, float aSampleRate)
+ : AudioNode(aContext, aIsOffline ? aNumberOfChannels : 2,
+ ChannelCountMode::Explicit, ChannelInterpretation::Speakers)
, mFramesToProduce(aLength)
, mAudioChannel(AudioChannel::Normal)
, mIsOffline(aIsOffline)
, mAudioChannelAgentPlaying(false)
, mExtraCurrentTime(0)
, mExtraCurrentTimeSinceLastStartedBlocking(0)
, mExtraCurrentTimeUpdatedSinceLastStableState(false)
+ , mCaptured(false)
{
bool startWithAudioDriver = true;
MediaStreamGraph* graph = aIsOffline ?
MediaStreamGraph::CreateNonRealtimeInstance(aSampleRate) :
MediaStreamGraph::GetInstance(startWithAudioDriver, aChannel);
AudioNodeEngine* engine = aIsOffline ?
new OfflineDestinationNodeEngine(this, aNumberOfChannels,
aLength, aSampleRate) :
@@ -505,23 +503,35 @@ AudioDestinationNode::WindowVolumeChange
return NS_OK;
}
NS_IMETHODIMP
AudioDestinationNode::WindowAudioCaptureChanged()
{
MOZ_ASSERT(mAudioChannelAgent);
- if (!mStream) {
+ if (!mStream || Context()->IsOffline()) {
return NS_OK;
}
- DebugOnly<bool> captured = GetOwner()->GetAudioCaptured();
+ bool captured = GetOwner()->GetAudioCaptured();
- // XXXtodopadenot actually capture
+ if (captured != mCaptured) {
+ if (captured) {
+ nsCOMPtr<nsPIDOMWindow> window = Context()->GetParentObject();
+ uint64_t id = window->WindowID();
+ mCaptureStreamPort =
+ mStream->Graph()->ConnectToCaptureStream(id, mStream);
+ } else {
+ mCaptureStreamPort->Disconnect();
+ mCaptureStreamPort->Destroy();
+ }
+ mCaptured = captured;
+ }
+
return NS_OK;
}
AudioChannel
AudioDestinationNode::MozAudioChannelType() const
{
return mAudioChannel;
}
@@ -694,13 +704,14 @@ AudioDestinationNode::InputMuted(bool aM
float volume = 0.0;
bool muted = true;
nsresult rv = mAudioChannelAgent->NotifyStartedPlaying(&volume, &muted);
if (NS_WARN_IF(NS_FAILED(rv))) {
return;
}
+ WindowAudioCaptureChanged();
WindowVolumeChanged(volume, muted);
}
} // namespace dom
} // namespace mozilla
--- a/dom/media/webaudio/AudioDestinationNode.h
+++ b/dom/media/webaudio/AudioDestinationNode.h
@@ -94,27 +94,29 @@ private:
void NotifyStableState();
void ScheduleStableStateNotification();
SelfReference<AudioDestinationNode> mOfflineRenderingRef;
uint32_t mFramesToProduce;
nsCOMPtr<nsIAudioChannelAgent> mAudioChannelAgent;
+ nsRefPtr<MediaInputPort> mCaptureStreamPort;
nsRefPtr<Promise> mOfflineRenderingPromise;
// Audio Channel Type.
AudioChannel mAudioChannel;
bool mIsOffline;
bool mAudioChannelAgentPlaying;
TimeStamp mStartedBlockingDueToBeingOnlyNode;
double mExtraCurrentTime;
double mExtraCurrentTimeSinceLastStartedBlocking;
bool mExtraCurrentTimeUpdatedSinceLastStableState;
+ bool mCaptured;
};
} // namespace dom
} // namespace mozilla
#endif