author | Andreea Pavel <apavel@mozilla.com> |
Fri, 20 Jul 2018 17:13:20 +0300 | |
changeset 821124 | 060df6ed6995bdba888551568560087805b902e4 |
parent 821123 | 666dc3b74320bbc91ab598427eef72423b1cfd3f |
child 821125 | 512f6987b0b2e4e6d85ac1554fac3dd5fafd9b3e |
push id | 117018 |
push user | bmo:sfoster@mozilla.com |
push date | Sat, 21 Jul 2018 04:05:10 +0000 |
bugs | 1476744, 1148354 |
milestone | 63.0a1 |
backs out | 0ae34db4c34ffb8f4f5dd5123acca6c24feaf72e 97d89ba5f93dad118fe54358ff70c28a3a33cde5 d9174c023701e47ee32db6dbb738a28f0a838dc1 b4085dba45c5daeab32ba237f5342ffd2dc78954 |
--- a/dom/media/webaudio/AudioBufferSourceNode.cpp +++ b/dom/media/webaudio/AudioBufferSourceNode.cpp @@ -664,16 +664,19 @@ AudioBufferSourceNode::Create(JSContext* void AudioBufferSourceNode::DestroyMediaStream() { bool hadStream = mStream; if (hadStream) { mStream->RemoveMainThreadListener(this); } AudioNode::DestroyMediaStream(); + if (hadStream && Context()) { + Context()->UnregisterAudioBufferSourceNode(this); + } } size_t AudioBufferSourceNode::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const { size_t amount = AudioNode::SizeOfExcludingThis(aMallocSizeOf); /* mBuffer can be shared and is accounted for separately. */
--- a/dom/media/webaudio/AudioContext.cpp +++ b/dom/media/webaudio/AudioContext.cpp @@ -88,16 +88,17 @@ NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN(Au NS_IMPL_CYCLE_COLLECTION_UNLINK(mListener) NS_IMPL_CYCLE_COLLECTION_UNLINK(mPromiseGripArray) NS_IMPL_CYCLE_COLLECTION_UNLINK(mPendingResumePromises) if (!tmp->mIsStarted) { NS_IMPL_CYCLE_COLLECTION_UNLINK(mActiveNodes) } // mDecodeJobs owns the WebAudioDecodeJob objects whose lifetime is managed explicitly. // mAllNodes is an array of weak pointers, ignore it here. + // mPannerNodes is an array of weak pointers, ignore it here. // mBasicWaveFormCache cannot participate in cycles, ignore it here. // Remove weak reference on the global window as the context is not usable // without mDestination. tmp->DisconnectFromWindow(); NS_IMPL_CYCLE_COLLECTION_UNLINK_END_INHERITED(DOMEventTargetHelper) NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(AudioContext, @@ -108,16 +109,17 @@ NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_ NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPendingResumePromises) if (!tmp->mIsStarted) { MOZ_ASSERT(tmp->mIsOffline, "Online AudioContexts should always be started"); NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mActiveNodes) } // mDecodeJobs owns the WebAudioDecodeJob objects whose lifetime is managed explicitly. // mAllNodes is an array of weak pointers, ignore it here. + // mPannerNodes is an array of weak pointers, ignore it here. // mBasicWaveFormCache cannot participate in cycles, ignore it here. NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END NS_IMPL_ADDREF_INHERITED(AudioContext, DOMEventTargetHelper) NS_IMPL_RELEASE_INHERITED(AudioContext, DOMEventTargetHelper) NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(AudioContext) NS_INTERFACE_MAP_ENTRY(nsIMemoryReporter) @@ -648,16 +650,39 @@ AudioContext::RegisterActiveNode(AudioNo } void AudioContext::UnregisterActiveNode(AudioNode* aNode) { mActiveNodes.RemoveEntry(aNode); } +void +AudioContext::UnregisterAudioBufferSourceNode(AudioBufferSourceNode* aNode) +{ + UpdatePannerSource(); +} + +void +AudioContext::UnregisterPannerNode(PannerNode* aNode) +{ + mPannerNodes.RemoveEntry(aNode); + if (mListener) { + mListener->UnregisterPannerNode(aNode); + } +} + +void +AudioContext::UpdatePannerSource() +{ + for (auto iter = mPannerNodes.Iter(); !iter.Done(); iter.Next()) { + iter.Get()->GetKey()->FindConnectedSources(); + } +} + uint32_t AudioContext::MaxChannelCount() const { return std::min<uint32_t>(WebAudioUtils::MaxChannelCount, mIsOffline ? mNumberOfChannels : CubebUtils::MaxNumberOfChannels()); } uint32_t @@ -1172,16 +1197,17 @@ AudioContext::SizeOfIncludingThis(mozill if (mListener) { amount += mListener->SizeOfIncludingThis(aMallocSizeOf); } amount += mDecodeJobs.ShallowSizeOfExcludingThis(aMallocSizeOf); for (uint32_t i = 0; i < mDecodeJobs.Length(); ++i) { amount += mDecodeJobs[i]->SizeOfIncludingThis(aMallocSizeOf); } amount += mActiveNodes.ShallowSizeOfExcludingThis(aMallocSizeOf); + amount += mPannerNodes.ShallowSizeOfExcludingThis(aMallocSizeOf); return amount; } NS_IMETHODIMP AudioContext::CollectReports(nsIHandleReportCallback* aHandleReport, nsISupports* aData, bool aAnonymize) { const nsLiteralCString
--- a/dom/media/webaudio/AudioContext.h +++ b/dom/media/webaudio/AudioContext.h @@ -304,16 +304,20 @@ public: // Nodes unregister when they have finished producing sound for the // foreseeable future. // Do NOT call UnregisterActiveNode from an AudioNode destructor. // If the destructor is called, then the Node has already been unregistered. // The destructor may be called during hashtable enumeration, during which // unregistering would not be safe. void UnregisterActiveNode(AudioNode* aNode); + void UnregisterAudioBufferSourceNode(AudioBufferSourceNode* aNode); + void UnregisterPannerNode(PannerNode* aNode); + void UpdatePannerSource(); + uint32_t MaxChannelCount() const; uint32_t ActiveNodeCount() const; void Mute() const; void Unmute() const; JSObject* GetGlobalJSObject() const; @@ -362,16 +366,19 @@ private: // is not allowed to play, the promise would be pending in this array and be // resolved until audio context has been allowed and user call resume() again. nsTArray<RefPtr<Promise>> mPendingResumePromises; // See RegisterActiveNode. These will keep the AudioContext alive while it // is rendering and the window remains alive. nsTHashtable<nsRefPtrHashKey<AudioNode> > mActiveNodes; // Raw (non-owning) references to all AudioNodes for this AudioContext. nsTHashtable<nsPtrHashKey<AudioNode> > mAllNodes; + // Hashsets containing all the PannerNodes, to compute the doppler shift. + // These are weak pointers. + nsTHashtable<nsPtrHashKey<PannerNode> > mPannerNodes; // Cache to avoid recomputing basic waveforms all the time. RefPtr<BasicWaveFormCache> mBasicWaveFormCache; // Number of channels passed in the OfflineAudioContext ctor. uint32_t mNumberOfChannels; bool mIsOffline; bool mIsStarted; bool mIsShutDown; // Close has been called, reject suspend and resume call.
--- a/dom/media/webaudio/AudioListener.cpp +++ b/dom/media/webaudio/AudioListener.cpp @@ -2,75 +2,33 @@ /* vim:set ts=2 sw=2 sts=2 et cindent: */ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "AudioListener.h" #include "AudioContext.h" #include "mozilla/dom/AudioListenerBinding.h" -#include "MediaStreamGraphImpl.h" namespace mozilla { namespace dom { NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(AudioListener, mContext) NS_IMPL_CYCLE_COLLECTION_ROOT_NATIVE(AudioListener, AddRef) NS_IMPL_CYCLE_COLLECTION_UNROOT_NATIVE(AudioListener, Release) -AudioListenerEngine::AudioListenerEngine() - : mPosition() - , mFrontVector(0., 0., -1.) - , mRightVector(1., 0., 0.) -{ -} - -void -AudioListenerEngine::RecvListenerEngineEvent( - AudioListenerEngine::AudioListenerParameter aParameter, - const ThreeDPoint& aValue) -{ - switch (aParameter) { - case AudioListenerParameter::POSITION: - mPosition = aValue; - break; - case AudioListenerParameter::FRONT: - mFrontVector = aValue; - break; - case AudioListenerParameter::RIGHT: - mRightVector = aValue; - break; - default: - MOZ_CRASH("Not handled"); - } -} - -const ThreeDPoint& -AudioListenerEngine::Position() const -{ - return mPosition; -} -const ThreeDPoint& -AudioListenerEngine::FrontVector() const -{ - return mFrontVector; -} -const ThreeDPoint& -AudioListenerEngine::RightVector() const -{ - return mRightVector; -} - AudioListener::AudioListener(AudioContext* aContext) : mContext(aContext) - , mEngine(MakeUnique<AudioListenerEngine>()) , mPosition() , mFrontVector(0., 0., -1.) , mRightVector(1., 0., 0.) + , mVelocity() + , mDopplerFactor(1.) + , mSpeedOfSound(343.3) // meters/second { MOZ_ASSERT(aContext); } JSObject* AudioListener::WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) { return AudioListener_Binding::Wrap(aCx, this, aGivenProto); @@ -97,72 +55,77 @@ AudioListener::SetOrientation(double aX, ThreeDPoint right = front.CrossProduct(up); if (right.IsZero()) { return; } right.Normalize(); if (!mFrontVector.FuzzyEqual(front)) { mFrontVector = front; - SendListenerEngineEvent(AudioListenerEngine::AudioListenerParameter::FRONT, - mFrontVector); + SendThreeDPointParameterToStream(PannerNode::LISTENER_FRONT_VECTOR, front); } if (!mRightVector.FuzzyEqual(right)) { mRightVector = right; - SendListenerEngineEvent(AudioListenerEngine::AudioListenerParameter::RIGHT, - mRightVector); + SendThreeDPointParameterToStream(PannerNode::LISTENER_RIGHT_VECTOR, right); } } void -AudioListener::SetPosition(double aX, double aY, double aZ) +AudioListener::RegisterPannerNode(PannerNode* aPannerNode) { - if (WebAudioUtils::FuzzyEqual(mPosition.x, aX) && - WebAudioUtils::FuzzyEqual(mPosition.y, aY) && - WebAudioUtils::FuzzyEqual(mPosition.z, aZ)) { - return; - } - mPosition.x = aX; - mPosition.y = aY; - mPosition.z = aZ; - SendListenerEngineEvent(AudioListenerEngine::AudioListenerParameter::POSITION, - mPosition); + mPanners.AppendElement(aPannerNode); + + // Let the panner node know about our parameters + aPannerNode->SendThreeDPointParameterToStream(PannerNode::LISTENER_POSITION, mPosition); + aPannerNode->SendThreeDPointParameterToStream(PannerNode::LISTENER_FRONT_VECTOR, mFrontVector); + aPannerNode->SendThreeDPointParameterToStream(PannerNode::LISTENER_RIGHT_VECTOR, mRightVector); + aPannerNode->SendThreeDPointParameterToStream(PannerNode::LISTENER_VELOCITY, mVelocity); + aPannerNode->SendDoubleParameterToStream(PannerNode::LISTENER_DOPPLER_FACTOR, mDopplerFactor); + aPannerNode->SendDoubleParameterToStream(PannerNode::LISTENER_SPEED_OF_SOUND, mSpeedOfSound); + UpdatePannersVelocity(); +} + +void AudioListener::UnregisterPannerNode(PannerNode* aPannerNode) +{ + mPanners.RemoveElement(aPannerNode); } void -AudioListener::SendListenerEngineEvent( - AudioListenerEngine::AudioListenerParameter aParameter, - const ThreeDPoint& aValue) +AudioListener::SendDoubleParameterToStream(uint32_t aIndex, double aValue) +{ + for (uint32_t i = 0; i < mPanners.Length(); ++i) { + if (mPanners[i]) { + mPanners[i]->SendDoubleParameterToStream(aIndex, aValue); + } + } +} + +void +AudioListener::SendThreeDPointParameterToStream(uint32_t aIndex, const ThreeDPoint& aValue) { - class Message final : public ControlMessage - { - public: - Message(AudioListenerEngine* aEngine, - AudioListenerEngine::AudioListenerParameter aParameter, - const ThreeDPoint& aValue) - : ControlMessage(nullptr) - , mEngine(aEngine) - , mParameter(aParameter) - , mValue(aValue) - { + for (uint32_t i = 0; i < mPanners.Length(); ++i) { + if (mPanners[i]) { + mPanners[i]->SendThreeDPointParameterToStream(aIndex, aValue); } - void Run() override - { - mEngine->RecvListenerEngineEvent(mParameter, mValue); + } +} + +void AudioListener::UpdatePannersVelocity() +{ + for (uint32_t i = 0; i < mPanners.Length(); ++i) { + if (mPanners[i]) { + mPanners[i]->SendDopplerToSourcesIfNeeded(); } - AudioListenerEngine* mEngine; - AudioListenerEngine::AudioListenerParameter mParameter; - ThreeDPoint mValue; - }; - - mContext->DestinationStream()->GraphImpl()->AppendMessage( - MakeUnique<Message>(mEngine.get(), aParameter, aValue)); + } } size_t AudioListener::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const { - return aMallocSizeOf(this); + size_t amount = aMallocSizeOf(this); + // AudioNodes are tracked separately + amount += mPanners.ShallowSizeOfExcludingThis(aMallocSizeOf); + return amount; } } // namespace dom } // namespace mozilla
--- a/dom/media/webaudio/AudioListener.h +++ b/dom/media/webaudio/AudioListener.h @@ -16,39 +16,16 @@ #include "WebAudioUtils.h" #include "js/TypeDecls.h" #include "mozilla/MemoryReporting.h" namespace mozilla { namespace dom { -class AudioListenerEngine final -{ -public: - enum class AudioListenerParameter - { - POSITION, - FRONT, // unit length - RIGHT // unit length, orthogonal to FRONT - }; - AudioListenerEngine(); - void RecvListenerEngineEvent( - AudioListenerEngine::AudioListenerParameter aParameter, - const ThreeDPoint& aValue); - const ThreeDPoint& Position() const; - const ThreeDPoint& FrontVector() const; - const ThreeDPoint& RightVector() const; - -private: - ThreeDPoint mPosition; - ThreeDPoint mFrontVector; - ThreeDPoint mRightVector; -}; - class AudioListener final : public nsWrapperCache { public: explicit AudioListener(AudioContext* aContext); NS_INLINE_DECL_CYCLE_COLLECTING_NATIVE_REFCOUNTING(AudioListener) NS_DECL_CYCLE_COLLECTION_SCRIPT_HOLDER_NATIVE_CLASS(AudioListener) @@ -56,35 +33,101 @@ public: AudioContext* GetParentObject() const { return mContext; } JSObject* WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) override; - void SetPosition(double aX, double aY, double aZ); + double DopplerFactor() const + { + return mDopplerFactor; + } + void SetDopplerFactor(double aDopplerFactor) + { + if (WebAudioUtils::FuzzyEqual(mDopplerFactor, aDopplerFactor)) { + return; + } + mDopplerFactor = aDopplerFactor; + SendDoubleParameterToStream(PannerNode::LISTENER_DOPPLER_FACTOR, mDopplerFactor); + } + + double SpeedOfSound() const + { + return mSpeedOfSound; + } + void SetSpeedOfSound(double aSpeedOfSound) + { + if (WebAudioUtils::FuzzyEqual(mSpeedOfSound, aSpeedOfSound)) { + return; + } + mSpeedOfSound = aSpeedOfSound; + SendDoubleParameterToStream(PannerNode::LISTENER_SPEED_OF_SOUND, mSpeedOfSound); + } + + void SetPosition(double aX, double aY, double aZ) + { + if (WebAudioUtils::FuzzyEqual(mPosition.x, aX) && + WebAudioUtils::FuzzyEqual(mPosition.y, aY) && + WebAudioUtils::FuzzyEqual(mPosition.z, aZ)) { + return; + } + mPosition.x = aX; + mPosition.y = aY; + mPosition.z = aZ; + SendThreeDPointParameterToStream(PannerNode::LISTENER_POSITION, mPosition); + } + + const ThreeDPoint& Position() const + { + return mPosition; + } + void SetOrientation(double aX, double aY, double aZ, double aXUp, double aYUp, double aZUp); - const AudioListenerEngine* Engine() { return mEngine.get(); } + const ThreeDPoint& Velocity() const + { + return mVelocity; + } + + void SetVelocity(double aX, double aY, double aZ) + { + if (WebAudioUtils::FuzzyEqual(mVelocity.x, aX) && + WebAudioUtils::FuzzyEqual(mVelocity.y, aY) && + WebAudioUtils::FuzzyEqual(mVelocity.z, aZ)) { + return; + } + mVelocity.x = aX; + mVelocity.y = aY; + mVelocity.z = aZ; + SendThreeDPointParameterToStream(PannerNode::LISTENER_VELOCITY, mVelocity); + UpdatePannersVelocity(); + } + + void RegisterPannerNode(PannerNode* aPannerNode); + void UnregisterPannerNode(PannerNode* aPannerNode); private: - void SendListenerEngineEvent( - AudioListenerEngine::AudioListenerParameter aParameter, - const ThreeDPoint& aValue); + ~AudioListener() {} - ~AudioListener() = default; - + void SendDoubleParameterToStream(uint32_t aIndex, double aValue); void SendThreeDPointParameterToStream(uint32_t aIndex, const ThreeDPoint& aValue); + void UpdatePannersVelocity(); + private: + friend class PannerNode; RefPtr<AudioContext> mContext; - const UniquePtr<AudioListenerEngine> mEngine; ThreeDPoint mPosition; ThreeDPoint mFrontVector; ThreeDPoint mRightVector; + ThreeDPoint mVelocity; + double mDopplerFactor; + double mSpeedOfSound; + nsTArray<WeakPtr<PannerNode> > mPanners; }; } // namespace dom } // namespace mozilla #endif
--- a/dom/media/webaudio/AudioNode.cpp +++ b/dom/media/webaudio/AudioNode.cpp @@ -239,16 +239,19 @@ AudioNode::Connect(AudioNode& aDestinati MOZ_ASSERT(aOutput <= UINT16_MAX, "Unexpected large output port number"); input->mStreamPort = destinationStream-> AllocateInputPort(mStream, AudioNodeStream::AUDIO_TRACK, TRACK_ANY, static_cast<uint16_t>(aInput), static_cast<uint16_t>(aOutput)); } aDestination.NotifyInputsChanged(); + // This connection may have connected a panner and a source. + Context()->UpdatePannerSource(); + return &aDestination; } void AudioNode::Connect(AudioParam& aDestination, uint32_t aOutput, ErrorResult& aRv) { if (aOutput >= NumberOfOutputs()) { @@ -446,16 +449,19 @@ AudioNode::Disconnect(ErrorResult& aRv) for (int32_t outputIndex = mOutputParams.Length() - 1; outputIndex >= 0; --outputIndex) { DisconnectMatchingDestinationInputs<AudioParam>(outputIndex, [](const InputNode&) { return true; }); } + + // This disconnection may have disconnected a panner and a source. + Context()->UpdatePannerSource(); } void AudioNode::Disconnect(uint32_t aOutput, ErrorResult& aRv) { if (aOutput >= NumberOfOutputs()) { aRv.Throw(NS_ERROR_DOM_INDEX_SIZE_ERR); return; @@ -473,16 +479,19 @@ AudioNode::Disconnect(uint32_t aOutput, for (int32_t outputIndex = mOutputParams.Length() - 1; outputIndex >= 0; --outputIndex) { DisconnectMatchingDestinationInputs<AudioParam>( outputIndex, [aOutput](const InputNode& aInputNode) { return aInputNode.mOutputPort == aOutput; }); } + + // This disconnection may have disconnected a panner and a source. + Context()->UpdatePannerSource(); } void AudioNode::Disconnect(AudioNode& aDestination, ErrorResult& aRv) { bool wasConnected = false; for (int32_t outputIndex = mOutputNodes.Length() - 1; @@ -496,16 +505,19 @@ AudioNode::Disconnect(AudioNode& aDestin return true; }); } if (!wasConnected) { aRv.Throw(NS_ERROR_DOM_INVALID_ACCESS_ERR); return; } + + // This disconnection may have disconnected a panner and a source. + Context()->UpdatePannerSource(); } void AudioNode::Disconnect(AudioNode& aDestination, uint32_t aOutput, ErrorResult& aRv) { if (aOutput >= NumberOfOutputs()) { @@ -527,16 +539,19 @@ AudioNode::Disconnect(AudioNode& aDestin return aInputNode.mOutputPort == aOutput; }); } if (!wasConnected) { aRv.Throw(NS_ERROR_DOM_INVALID_ACCESS_ERR); return; } + + // This disconnection may have disconnected a panner and a source. + Context()->UpdatePannerSource(); } void AudioNode::Disconnect(AudioNode& aDestination, uint32_t aOutput, uint32_t aInput, ErrorResult& aRv) { @@ -565,16 +580,19 @@ AudioNode::Disconnect(AudioNode& aDestin aInputNode.mInputPort == aInput; }); } if (!wasConnected) { aRv.Throw(NS_ERROR_DOM_INVALID_ACCESS_ERR); return; } + + // This disconnection may have disconnected a panner and a source. + Context()->UpdatePannerSource(); } void AudioNode::Disconnect(AudioParam& aDestination, ErrorResult& aRv) { bool wasConnected = false; for (int32_t outputIndex = mOutputParams.Length() - 1;
--- a/dom/media/webaudio/PannerNode.cpp +++ b/dom/media/webaudio/PannerNode.cpp @@ -22,53 +22,57 @@ using WebCore::HRTFPanner; namespace mozilla { namespace dom { using namespace std; NS_IMPL_CYCLE_COLLECTION_CLASS(PannerNode) NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(PannerNode, AudioNode) + if (tmp->Context()) { + tmp->Context()->UnregisterPannerNode(tmp); + } NS_IMPL_CYCLE_COLLECTION_UNLINK(mPositionX, mPositionY, mPositionZ, mOrientationX, mOrientationY, mOrientationZ) NS_IMPL_CYCLE_COLLECTION_UNLINK_END NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(PannerNode, AudioNode) NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mPositionX, mPositionY, mPositionZ, mOrientationX, mOrientationY, mOrientationZ) NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(PannerNode) NS_INTERFACE_MAP_END_INHERITING(AudioNode) NS_IMPL_ADDREF_INHERITED(PannerNode, AudioNode) NS_IMPL_RELEASE_INHERITED(PannerNode, AudioNode) class PannerNodeEngine final : public AudioNodeEngine { public: - explicit PannerNodeEngine(AudioNode* aNode, - AudioDestinationNode* aDestination, - const AudioListenerEngine* aListenerEngine) + explicit PannerNodeEngine(AudioNode* aNode, AudioDestinationNode* aDestination) : AudioNodeEngine(aNode) , mDestination(aDestination->Stream()) - , mListenerEngine(aListenerEngine) - // Please keep these default values consistent with PannerNode::PannerNode - // below. + // Please keep these default values consistent with PannerNode::PannerNode below. , mPanningModelFunction(&PannerNodeEngine::EqualPowerPanningFunction) , mDistanceModelFunction(&PannerNodeEngine::InverseGainFunction) , mPositionX(0.) , mPositionY(0.) , mPositionZ(0.) , mOrientationX(1.) , mOrientationY(0.) , mOrientationZ(0.) + , mVelocity() , mRefDistance(1.) , mMaxDistance(10000.) , mRolloffFactor(1.) , mConeInnerAngle(360.) , mConeOuterAngle(360.) , mConeOuterGain(0.) + // These will be initialized when a PannerNode is created, so just initialize them + // to some dummy values here. + , mListenerDopplerFactor(0.) + , mListenerSpeedOfSound(0.) , mLeftOverData(INT_MIN) { } void RecvTimelineEvent(uint32_t aIndex, AudioTimelineEvent& aEvent) override { MOZ_ASSERT(mDestination); WebAudioUtils::ConvertAudioTimelineEventToTicks(aEvent, @@ -143,33 +147,40 @@ public: break; default: NS_ERROR("Bad PannerNodeEngine Int32Parameter"); } } void SetThreeDPointParameter(uint32_t aIndex, const ThreeDPoint& aParam) override { switch (aIndex) { + case PannerNode::LISTENER_POSITION: mListenerPosition = aParam; break; + case PannerNode::LISTENER_FRONT_VECTOR: mListenerFrontVector = aParam; break; + case PannerNode::LISTENER_RIGHT_VECTOR: mListenerRightVector = aParam; break; + case PannerNode::LISTENER_VELOCITY: mListenerVelocity = aParam; break; case PannerNode::POSITION: mPositionX.SetValue(aParam.x); mPositionY.SetValue(aParam.y); mPositionZ.SetValue(aParam.z); break; case PannerNode::ORIENTATION: mOrientationX.SetValue(aParam.x); mOrientationY.SetValue(aParam.y); mOrientationZ.SetValue(aParam.z); break; + case PannerNode::VELOCITY: mVelocity = aParam; break; default: NS_ERROR("Bad PannerNodeEngine ThreeDPointParameter"); } } void SetDoubleParameter(uint32_t aIndex, double aParam) override { switch (aIndex) { + case PannerNode::LISTENER_DOPPLER_FACTOR: mListenerDopplerFactor = aParam; break; + case PannerNode::LISTENER_SPEED_OF_SOUND: mListenerSpeedOfSound = aParam; break; case PannerNode::REF_DISTANCE: mRefDistance = aParam; break; case PannerNode::MAX_DISTANCE: mMaxDistance = aParam; break; case PannerNode::ROLLOFF_FACTOR: mRolloffFactor = aParam; break; case PannerNode::CONE_INNER_ANGLE: mConeInnerAngle = aParam; break; case PannerNode::CONE_OUTER_ANGLE: mConeOuterAngle = aParam; break; case PannerNode::CONE_OUTER_GAIN: mConeOuterGain = aParam; break; default: NS_ERROR("Bad PannerNodeEngine DoubleParameter"); @@ -251,35 +262,39 @@ public: return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); } RefPtr<AudioNodeStream> mDestination; // This member is set on the main thread, but is not accessed on the rendering // thread untile mPanningModelFunction has changed, and this happens strictly // later, via a MediaStreamGraph ControlMessage. nsAutoPtr<HRTFPanner> mHRTFPanner; - // This is set in the ctor, and guaranteed to live longer than this engine: - // its lifetime is the same as the AudioContext itself. - const AudioListenerEngine* mListenerEngine; typedef void (PannerNodeEngine::*PanningModelFunction)(const AudioBlock& aInput, AudioBlock* aOutput, StreamTime tick); PanningModelFunction mPanningModelFunction; typedef float (PannerNodeEngine::*DistanceModelFunction)(double aDistance); DistanceModelFunction mDistanceModelFunction; AudioParamTimeline mPositionX; AudioParamTimeline mPositionY; AudioParamTimeline mPositionZ; AudioParamTimeline mOrientationX; AudioParamTimeline mOrientationY; AudioParamTimeline mOrientationZ; + ThreeDPoint mVelocity; double mRefDistance; double mMaxDistance; double mRolloffFactor; double mConeInnerAngle; double mConeOuterAngle; double mConeOuterGain; + ThreeDPoint mListenerPosition; + ThreeDPoint mListenerFrontVector; + ThreeDPoint mListenerRightVector; + ThreeDPoint mListenerVelocity; + double mListenerDopplerFactor; + double mListenerSpeedOfSound; int mLeftOverData; }; PannerNode::PannerNode(AudioContext* aContext) : AudioNode(aContext, 2, ChannelCountMode::Clamped_max, ChannelInterpretation::Speakers) @@ -287,29 +302,37 @@ PannerNode::PannerNode(AudioContext* aCo , mPanningModel(PanningModelType::Equalpower) , mDistanceModel(DistanceModelType::Inverse) , mPositionX(new AudioParam(this, PannerNode::POSITIONX, this->NodeType(), 0.f)) , mPositionY(new AudioParam(this, PannerNode::POSITIONY, this->NodeType(), 0.f)) , mPositionZ(new AudioParam(this, PannerNode::POSITIONZ, this->NodeType(), 0.f)) , mOrientationX(new AudioParam(this, PannerNode::ORIENTATIONX, this->NodeType(), 1.0f)) , mOrientationY(new AudioParam(this, PannerNode::ORIENTATIONY, this->NodeType(), 0.f)) , mOrientationZ(new AudioParam(this, PannerNode::ORIENTATIONZ, this->NodeType(), 0.f)) + , mVelocity() , mRefDistance(1.) , mMaxDistance(10000.) , mRolloffFactor(1.) , mConeInnerAngle(360.) , mConeOuterAngle(360.) , mConeOuterGain(0.) { - mStream = AudioNodeStream::Create( - aContext, - new PannerNodeEngine( - this, aContext->Destination(), aContext->Listener()->Engine()), - AudioNodeStream::NO_STREAM_FLAGS, - aContext->Graph()); + mStream = AudioNodeStream::Create(aContext, + new PannerNodeEngine(this, aContext->Destination()), + AudioNodeStream::NO_STREAM_FLAGS, + aContext->Graph()); + // We should register once we have set up our stream and engine. + Context()->Listener()->RegisterPannerNode(this); +} + +PannerNode::~PannerNode() +{ + if (Context()) { + Context()->UnregisterPannerNode(this); + } } /* static */ already_AddRefed<PannerNode> PannerNode::Create(AudioContext& aAudioContext, const PannerOptions& aOptions, ErrorResult& aRv) { if (aAudioContext.CheckClosed(aRv)) { @@ -349,31 +372,41 @@ void PannerNode::SetPanningModel(Panning static_cast<PannerNodeEngine*>(mStream->Engine())->CreateHRTFPanner(); } SendInt32ParameterToStream(PANNING_MODEL, int32_t(mPanningModel)); } size_t PannerNode::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const { - return AudioNode::SizeOfExcludingThis(aMallocSizeOf); + size_t amount = AudioNode::SizeOfExcludingThis(aMallocSizeOf); + amount += mSources.ShallowSizeOfExcludingThis(aMallocSizeOf); + return amount; } size_t PannerNode::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const { return aMallocSizeOf(this) + SizeOfExcludingThis(aMallocSizeOf); } JSObject* PannerNode::WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) { return PannerNode_Binding::Wrap(aCx, this, aGivenProto); } +void PannerNode::DestroyMediaStream() +{ + if (Context()) { + Context()->UnregisterPannerNode(this); + } + AudioNode::DestroyMediaStream(); +} + // Those three functions are described in the spec. float PannerNodeEngine::LinearGainFunction(double aDistance) { return 1 - mRolloffFactor * (std::max(std::min(aDistance, mMaxDistance), mRefDistance) - mRefDistance) / (mMaxDistance - mRefDistance); } float @@ -441,18 +474,20 @@ PannerNodeEngine::EqualPowerPanningFunct ThreeDPoint position = ConvertAudioParamTimelineTo3DP(mPositionX, mPositionY, mPositionZ, tick); ThreeDPoint orientation = ConvertAudioParamTimelineTo3DP(mOrientationX, mOrientationY, mOrientationZ, tick); if (!orientation.IsZero()) { orientation.Normalize(); } // For a stereo source, when both the listener and the panner are in // the same spot, and no cone gain is specified, this node is noop. - if (inputChannels == 2 && mListenerEngine->Position() == position && - mConeInnerAngle == 360 && mConeOuterAngle == 360) { + if (inputChannels == 2 && + mListenerPosition == position && + mConeInnerAngle == 360 && + mConeOuterAngle == 360) { *aOutput = aInput; return; } // The output of this node is always stereo, no matter what the inputs are. aOutput->AllocateChannels(2); ComputeAzimuthAndElevation(position, azimuth, elevation); @@ -601,28 +636,28 @@ PannerNodeEngine::EqualPowerPanningFunct AudioBlockInPlaceScale(outputR, alignedGain); } } // This algorithm is specified in the webaudio spec. void PannerNodeEngine::ComputeAzimuthAndElevation(const ThreeDPoint& position, float& aAzimuth, float& aElevation) { - ThreeDPoint sourceListener = position - mListenerEngine->Position(); + ThreeDPoint sourceListener = position - mListenerPosition; if (sourceListener.IsZero()) { aAzimuth = 0.0; aElevation = 0.0; return; } sourceListener.Normalize(); // Project the source-listener vector on the x-z plane. - const ThreeDPoint& listenerFront = mListenerEngine->FrontVector(); - const ThreeDPoint& listenerRight = mListenerEngine->RightVector(); + const ThreeDPoint& listenerFront = mListenerFrontVector; + const ThreeDPoint& listenerRight = mListenerRightVector; ThreeDPoint up = listenerRight.CrossProduct(listenerFront); double upProjection = sourceListener.DotProduct(up); aElevation = 90 - 180 * acos(upProjection) / M_PI; if (aElevation > 90) { aElevation = 180 - aElevation; } else if (aElevation < -90) { @@ -661,17 +696,17 @@ PannerNodeEngine::ComputeConeGain(const const ThreeDPoint& orientation) { // Omnidirectional source if (orientation.IsZero() || ((mConeInnerAngle == 360) && (mConeOuterAngle == 360))) { return 1; } // Normalized source-listener vector - ThreeDPoint sourceToListener = mListenerEngine->Position() - position; + ThreeDPoint sourceToListener = mListenerPosition - position; sourceToListener.Normalize(); // Angle between the source orientation vector and the source-listener vector double dotProduct = sourceToListener.DotProduct(orientation); double angle = 180 * acos(dotProduct) / M_PI; double absAngle = fabs(angle); // Divide by 2 here since API is entire angle (not half-angle) @@ -693,15 +728,103 @@ PannerNodeEngine::ComputeConeGain(const } return gain; } double PannerNodeEngine::ComputeDistanceGain(const ThreeDPoint& position) { - ThreeDPoint distanceVec = position - mListenerEngine->Position(); + ThreeDPoint distanceVec = position - mListenerPosition; float distance = sqrt(distanceVec.DotProduct(distanceVec)); return std::max(0.0f, (this->*mDistanceModelFunction)(distance)); } +float +PannerNode::ComputeDopplerShift() +{ + double dopplerShift = 1.0; // Initialize to default value + + AudioListener* listener = Context()->Listener(); + + if (listener->DopplerFactor() > 0) { + // Don't bother if both source and listener have no velocity. + if (!mVelocity.IsZero() || !listener->Velocity().IsZero()) { + // Calculate the source to listener vector. + ThreeDPoint sourceToListener = ConvertAudioParamTo3DP(mPositionX, mPositionY, mPositionZ) - listener->Velocity(); + + double sourceListenerMagnitude = sourceToListener.Magnitude(); + + double listenerProjection = sourceToListener.DotProduct(listener->Velocity()) / sourceListenerMagnitude; + double sourceProjection = sourceToListener.DotProduct(mVelocity) / sourceListenerMagnitude; + + listenerProjection = -listenerProjection; + sourceProjection = -sourceProjection; + + double scaledSpeedOfSound = listener->SpeedOfSound() / listener->DopplerFactor(); + listenerProjection = min(listenerProjection, scaledSpeedOfSound); + sourceProjection = min(sourceProjection, scaledSpeedOfSound); + + dopplerShift = ((listener->SpeedOfSound() - listener->DopplerFactor() * listenerProjection) / (listener->SpeedOfSound() - listener->DopplerFactor() * sourceProjection)); + + WebAudioUtils::FixNaN(dopplerShift); // Avoid illegal values + + // Limit the pitch shifting to 4 octaves up and 3 octaves down. + dopplerShift = min(dopplerShift, 16.); + dopplerShift = max(dopplerShift, 0.125); + } + } + + return dopplerShift; +} + +void +PannerNode::FindConnectedSources() +{ + mSources.Clear(); + std::set<AudioNode*> cycleSet; + FindConnectedSources(this, mSources, cycleSet); +} + +void +PannerNode::FindConnectedSources(AudioNode* aNode, + nsTArray<AudioBufferSourceNode*>& aSources, + std::set<AudioNode*>& aNodesSeen) +{ + if (!aNode) { + return; + } + + const nsTArray<InputNode>& inputNodes = aNode->InputNodes(); + + for(unsigned i = 0; i < inputNodes.Length(); i++) { + // Return if we find a node that we have seen already. + if (aNodesSeen.find(inputNodes[i].mInputNode) != aNodesSeen.end()) { + return; + } + aNodesSeen.insert(inputNodes[i].mInputNode); + // Recurse + FindConnectedSources(inputNodes[i].mInputNode, aSources, aNodesSeen); + + // Check if this node is an AudioBufferSourceNode that still have a stream, + // which means it has not finished playing. + AudioBufferSourceNode* node = inputNodes[i].mInputNode->AsAudioBufferSourceNode(); + if (node && node->GetStream()) { + aSources.AppendElement(node); + } + } +} + +void +PannerNode::SendDopplerToSourcesIfNeeded() +{ + // Don't bother sending the doppler shift if both the source and the listener + // are not moving, because the doppler shift is going to be 1.0. + if (!(Context()->Listener()->Velocity().IsZero() && mVelocity.IsZero())) { + for(uint32_t i = 0; i < mSources.Length(); i++) { + mSources[i]->SendDopplerShiftToStream(ComputeDopplerShift()); + } + } +} + + } // namespace dom } // namespace mozilla
--- a/dom/media/webaudio/PannerNode.h +++ b/dom/media/webaudio/PannerNode.h @@ -36,16 +36,18 @@ public: Constructor(const GlobalObject& aGlobal, AudioContext& aAudioContext, const PannerOptions& aOptions, ErrorResult& aRv) { return Create(aAudioContext, aOptions, aRv); } JSObject* WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) override; + void DestroyMediaStream() override; + void SetChannelCount(uint32_t aChannelCount, ErrorResult& aRv) override { if (aChannelCount > 2) { aRv.Throw(NS_ERROR_DOM_NOT_SUPPORTED_ERR); return; } AudioNode::SetChannelCount(aChannelCount, aRv); } @@ -98,16 +100,30 @@ public: return; } mOrientationX->SetValue(aX); mOrientationY->SetValue(aY); mOrientationZ->SetValue(aZ); SendThreeDPointParameterToStream(ORIENTATION, ConvertAudioParamTo3DP(mOrientationX, mOrientationY, mOrientationZ)); } + void SetVelocity(double aX, double aY, double aZ) + { + if (WebAudioUtils::FuzzyEqual(mVelocity.x, aX) && + WebAudioUtils::FuzzyEqual(mVelocity.y, aY) && + WebAudioUtils::FuzzyEqual(mVelocity.z, aZ)) { + return; + } + mVelocity.x = aX; + mVelocity.y = aY; + mVelocity.z = aZ; + SendThreeDPointParameterToStream(VELOCITY, mVelocity); + SendDopplerToSourcesIfNeeded(); + } + double RefDistance() const { return mRefDistance; } void SetRefDistance(double aRefDistance) { if (WebAudioUtils::FuzzyEqual(mRefDistance, aRefDistance)) { return; @@ -206,41 +222,54 @@ public: return mOrientationY; } AudioParam* OrientationZ() { return mOrientationZ; } + + float ComputeDopplerShift(); + void SendDopplerToSourcesIfNeeded(); + void FindConnectedSources(); + void FindConnectedSources(AudioNode* aNode, nsTArray<AudioBufferSourceNode*>& aSources, std::set<AudioNode*>& aSeenNodes); + const char* NodeType() const override { return "PannerNode"; } size_t SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) const override; size_t SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) const override; private: explicit PannerNode(AudioContext* aContext); - ~PannerNode() = default; + ~PannerNode(); friend class AudioListener; friend class PannerNodeEngine; enum EngineParameters { + LISTENER_POSITION, + LISTENER_FRONT_VECTOR, // unit length + LISTENER_RIGHT_VECTOR, // unit length, orthogonal to LISTENER_FRONT_VECTOR + LISTENER_VELOCITY, + LISTENER_DOPPLER_FACTOR, + LISTENER_SPEED_OF_SOUND, PANNING_MODEL, DISTANCE_MODEL, POSITION, POSITIONX, POSITIONY, POSITIONZ, ORIENTATION, // unit length or zero ORIENTATIONX, ORIENTATIONY, ORIENTATIONZ, + VELOCITY, REF_DISTANCE, MAX_DISTANCE, ROLLOFF_FACTOR, CONE_INNER_ANGLE, CONE_OUTER_ANGLE, CONE_OUTER_GAIN }; @@ -252,21 +281,26 @@ private: PanningModelType mPanningModel; DistanceModelType mDistanceModel; RefPtr<AudioParam> mPositionX; RefPtr<AudioParam> mPositionY; RefPtr<AudioParam> mPositionZ; RefPtr<AudioParam> mOrientationX; RefPtr<AudioParam> mOrientationY; RefPtr<AudioParam> mOrientationZ; + ThreeDPoint mVelocity; double mRefDistance; double mMaxDistance; double mRolloffFactor; double mConeInnerAngle; double mConeOuterAngle; double mConeOuterGain; + + // An array of all the AudioBufferSourceNode connected directly or indirectly + // to this AudioPannerNode. + nsTArray<AudioBufferSourceNode*> mSources; }; } // namespace dom } // namespace mozilla #endif
--- a/dom/media/webaudio/test/mochitest.ini +++ b/dom/media/webaudio/test/mochitest.ini @@ -88,16 +88,17 @@ tags=capturestream [test_bug839753.html] [test_bug845960.html] [test_bug856771.html] [test_bug866570.html] [test_bug866737.html] [test_bug867089.html] [test_bug867104.html] [test_bug867174.html] +[test_bug867203.html] [test_bug875221.html] [test_bug875402.html] [test_bug894150.html] [test_bug956489.html] [test_bug964376.html] [test_bug966247.html] tags=capturestream [test_bug972678.html]
--- a/dom/media/webaudio/test/test_AudioListener.html +++ b/dom/media/webaudio/test/test_AudioListener.html @@ -8,21 +8,28 @@ <body> <pre id="test"> <script class="testbody" type="text/javascript"> SimpleTest.waitForExplicitFinish(); addLoadEvent(function() { var context = new AudioContext(); ok("listener" in context, "AudioContext.listener should exist"); + ok(Math.abs(context.listener.dopplerFactor - 1.0) < 1e-4, "Correct default doppler factor"); + ok(Math.abs(context.listener.speedOfSound - 343.3) < 1e-4, "Correct default speed of sound value"); + context.listener.dopplerFactor = 0.5; + ok(Math.abs(context.listener.dopplerFactor - 0.5) < 1e-4, "The doppler factor value can be changed"); + context.listener.speedOfSound = 400; + ok(Math.abs(context.listener.speedOfSound - 400) < 1e-4, "The speed of sound can be changed"); // The values set by the following cannot be read from script, but the // implementation is simple enough, so we just make sure that nothing throws. with (context.listener) { setPosition(1.0, 1.0, 1.0); setOrientation(1.0, 1.0, 1.0, 1.0, 1.0, 1.0); + setVelocity(0.5, 1.0, 1.5); } SimpleTest.finish(); }); </script> </pre> </body> </html>
new file mode 100644 --- /dev/null +++ b/dom/media/webaudio/test/test_bug867203.html @@ -0,0 +1,34 @@ +<!DOCTYPE HTML> +<html> +<head> + <title>Crashtest for bug 867203</title> + <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" /> +</head> +<body> +<pre id="test"> +<script class="testbody" type="text/javascript"> + +SimpleTest.waitForExplicitFinish(); +addLoadEvent(function() { + var ctx = new AudioContext(); + + var panner1 = ctx.createPanner(); + panner1.setVelocity(1, 1, 1); + ctx.listener.setVelocity(1, 1, 1); + (function() { + ctx.createBufferSource().connect(panner1); + })(); + SpecialPowers.forceGC(); + SpecialPowers.forceCC(); + ctx.createPanner(); + + ok(true, "We did not crash."); + SimpleTest.finish(); +}); + + +</script> +</pre> +</body> +</html>
--- a/dom/media/webaudio/test/test_pannerNode.html +++ b/dom/media/webaudio/test/test_pannerNode.html @@ -50,16 +50,18 @@ addLoadEvent(function() { near(panner.positionY.value, 1, "setPosition sets AudioParam properly"); near(panner.positionZ.value, 1, "setPosition sets AudioParam properly"); panner.setOrientation(0, 1, 0); near(panner.orientationX.value, 0, "setOrientation sets AudioParam properly"); near(panner.orientationY.value, 1, "setOrientation sets AudioParam properly"); near(panner.orientationZ.value, 0, "setOrientation sets AudioParam properly"); + panner.setVelocity(1, 1, 1); + source.start(0); SimpleTest.executeSoon(function() { source.stop(0); source.disconnect(); panner.disconnect(); SimpleTest.finish(); });
--- a/dom/webidl/AudioListener.webidl +++ b/dom/webidl/AudioListener.webidl @@ -7,13 +7,25 @@ * https://webaudio.github.io/web-audio-api/ * * Copyright © 2012 W3C® (MIT, ERCIM, Keio), All Rights Reserved. W3C * liability, trademark and document use rules apply. */ [Pref="dom.webaudio.enabled"] interface AudioListener { + + // same as OpenAL (default 1) + [Deprecated="PannerNodeDoppler"] + attribute double dopplerFactor; + + // in meters / second (default 343.3) + [Deprecated="PannerNodeDoppler"] + attribute double speedOfSound; + // Uses a 3D cartesian coordinate system void setPosition(double x, double y, double z); void setOrientation(double x, double y, double z, double xUp, double yUp, double zUp); + [Deprecated="PannerNodeDoppler"] + void setVelocity(double x, double y, double z); + };
--- a/dom/webidl/PannerNode.webidl +++ b/dom/webidl/PannerNode.webidl @@ -43,16 +43,18 @@ dictionary PannerOptions : AudioNodeOpti interface PannerNode : AudioNode { // Default for stereo is equalpower attribute PanningModelType panningModel; // Uses a 3D cartesian coordinate system void setPosition(double x, double y, double z); void setOrientation(double x, double y, double z); + [Deprecated="PannerNodeDoppler"] + void setVelocity(double x, double y, double z); // Cartesian coordinate for position readonly attribute AudioParam positionX; readonly attribute AudioParam positionY; readonly attribute AudioParam positionZ; // Cartesian coordinate for orientation readonly attribute AudioParam orientationX;
--- a/testing/web-platform/tests/webaudio/historical.html +++ b/testing/web-platform/tests/webaudio/historical.html @@ -7,23 +7,9 @@ "webkitAudioContext", "webkitAudioPannerNode", "webkitOfflineAudioContext", ].forEach(name => { test(function() { assert_false(name in window); }, name + " interface should not exist"); }); - -[ - "dopplerFactor", - "speedOfSound", - "setVelocity" -].forEach(name => { - test(function() { - assert_false(name in AudioListener.prototype); - }, name + " member should not exist on the AudioListener."); -}); - -test(function() { - assert_false("setVelocity" in PannerNode.prototype); -}, "setVelocity should not exist on PannerNodes."); </script>