author | Alex Chronopoulos <achronop@gmail.com> |
Thu, 18 Jun 2020 13:58:06 +0000 | |
changeset 536550 | 8d4faaf6604cb161eaecc1e63ea00d8ec813ec91 |
parent 536549 | 76ff384f613cc3c71d58dfade8bda26f899778a2 |
child 536551 | 49c83d2045f1f63b6558c83b095acdbb6940e329 |
push id | 37530 |
push user | nbeleuzu@mozilla.com |
push date | Mon, 22 Jun 2020 21:47:58 +0000 |
treeherder | mozilla-central@9b38f4b9d883 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | padenot |
bugs | 1493990 |
milestone | 79.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/dom/media/AudioDriftCorrection.h +++ b/dom/media/AudioDriftCorrection.h @@ -162,30 +162,35 @@ class AudioDriftCorrection final { * The source audio frames and request the number of target audio frames must * be provided. The duration of the source and the output is considered as the * source clock and the target clock. The input is buffered internally so some * latency exists. The returned AudioSegment must be cleaned up because the * internal buffer will be reused after 100ms. If the drift correction (and * possible resampling) is not possible due to lack of input data an empty * AudioSegment will be returned. Not thread-safe. */ - AudioSegment RequestFrames(const AudioSegment& aInput, int aOutputFrames) { + AudioSegment RequestFrames(const AudioSegment& aInput, + int32_t aOutputFrames) { // Very important to go first since the Dynamic will get the sample format // from the chunk. if (aInput.GetDuration()) { // Always go through the resampler because the clock might shift later. mResampler.AppendInput(aInput); } mClockDrift.UpdateClock(aInput.GetDuration(), aOutputFrames, mResampler.InputDuration()); TrackRate receivingRate = mTargetRate * mClockDrift.GetCorrection(); // Update resampler's rate if there is a new correction. mResampler.UpdateOutRate(receivingRate); // If it does not have enough frames the result will be an empty segment. - return mResampler.Resample(aOutputFrames); + AudioSegment output = mResampler.Resample(aOutputFrames); + if (output.IsEmpty()) { + output.AppendNullData(aOutputFrames); + } + return output; } private: ClockDrift mClockDrift; AudioResampler mResampler; const int32_t mTargetRate; };
new file mode 100644 --- /dev/null +++ b/dom/media/CrossGraphTrack.cpp @@ -0,0 +1,197 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "CrossGraphTrack.h" + +#include "AudioDeviceInfo.h" +#include "AudioStreamTrack.h" +#include "MediaTrackGraphImpl.h" +#include "mozilla/Logging.h" + +namespace mozilla { + +#ifdef LOG +# undef LOG +#endif +#ifdef LOG_TEST +# undef LOG_TEST +#endif + +extern LazyLogModule gForwardedInputTrackLog; +#define LOG(type, msg) MOZ_LOG(gForwardedInputTrackLog, type, msg) +#define LOG_TEST(type) MOZ_LOG_TEST(gForwardedInputTrackLog, type) + +CrossGraphManager* CrossGraphManager::Connect( + const RefPtr<dom::AudioStreamTrack>& aStreamTrack, AudioDeviceInfo* aSink, + nsPIDOMWindowInner* aWindow) { + uint32_t defaultRate; + aSink->GetDefaultRate(&defaultRate); + LOG(LogLevel::Debug, + ("CrossGraphManager::Connect: sink id: %p at rate %u, primary rate %d", + aSink->DeviceID(), defaultRate, aStreamTrack->Graph()->GraphRate())); + + if (!aSink->DeviceID()) { + return nullptr; + } + + MediaTrackGraph* newGraph = + MediaTrackGraph::GetInstance(MediaTrackGraph::AUDIO_THREAD_DRIVER, + aWindow, defaultRate, aSink->DeviceID()); + + return CrossGraphManager::Connect(aStreamTrack, newGraph); +} + +CrossGraphManager* CrossGraphManager::Connect( + const RefPtr<dom::AudioStreamTrack>& aStreamTrack, + MediaTrackGraph* aPartnerGraph) { + if (aStreamTrack->Graph() == aPartnerGraph) { + // Primary graph the same as partner graph, just remove the existing cross + // graph connection + return nullptr; + } + + RefPtr<CrossGraphReceiver> receiver = aPartnerGraph->CreateCrossGraphReceiver( + aStreamTrack->Graph()->GraphRate()); + + RefPtr<CrossGraphTransmitter> transmitter = + aStreamTrack->Graph()->CreateCrossGraphTransmitter(receiver); + + RefPtr<MediaInputPort> port = + aStreamTrack->ForwardTrackContentsTo(transmitter); + + return new CrossGraphManager(port); +} + +RefPtr<CrossGraphTransmitter> CrossGraphManager::GetTransmitter() { + MOZ_ASSERT(mSourcePort); + return mSourcePort->GetDestination()->AsCrossGraphTransmitter(); +} + +RefPtr<CrossGraphReceiver> CrossGraphManager::GetReceiver() { + MOZ_ASSERT(mSourcePort); + return GetTransmitter()->mReceiver; +} + +void CrossGraphManager::AddAudioOutput(void* aKey) { + MOZ_ASSERT(GetTransmitter()); + GetReceiver()->AddAudioOutput(aKey); +} + +void CrossGraphManager::RemoveAudioOutput(void* aKey) { + MOZ_ASSERT(GetTransmitter()); + GetReceiver()->RemoveAudioOutput(aKey); +} + +void CrossGraphManager::SetAudioOutputVolume(void* aKey, float aVolume) { + MOZ_ASSERT(GetTransmitter()); + GetReceiver()->SetAudioOutputVolume(aKey, aVolume); +} + +void CrossGraphManager::Destroy() { + MOZ_ASSERT(GetTransmitter()); + GetTransmitter()->Destroy(); + mSourcePort->Destroy(); + mSourcePort = nullptr; +} + +RefPtr<GenericPromise> CrossGraphManager::EnsureConnected() { + // The primary graph is already working check the partner (receiver's) graph. + return GetReceiver()->Graph()->NotifyWhenDeviceStarted(GetReceiver().get()); +} + +/** CrossGraphTransmitter **/ + +CrossGraphTransmitter::CrossGraphTransmitter(TrackRate aSampleRate, + CrossGraphReceiver* aReceiver) + : ForwardedInputTrack(aSampleRate, MediaSegment::AUDIO), + mReceiver(aReceiver) {} + +void CrossGraphTransmitter::ProcessInput(GraphTime aFrom, GraphTime aTo, + uint32_t aFlags) { + // Get previous end before mSegment is updated. + TrackTime previousEnd = GetEnd(); + // Update mSegment from source track. + ForwardedInputTrack::ProcessInput(aFrom, aTo, aFlags); + + MOZ_ASSERT(mInputPort); + if (mInputPort->GetSource()->Ended() && + (mInputPort->GetSource()->GetEnd() <= + mInputPort->GetSource()->GraphTimeToTrackTimeWithBlocking(aTo))) { + return; + } + + LOG(LogLevel::Verbose, + ("Transmitter (%p) mSegment: duration: %" PRId64 ", from %" PRId64 + ", to %" PRId64 ", ticks %" PRId64 "", + this, mSegment->GetDuration(), aFrom, aTo, aTo - aFrom)); + + AudioSegment* audio = GetData<AudioSegment>(); + TrackTime ticks = aTo - aFrom; + MOZ_ASSERT(ticks); + MOZ_ASSERT(audio->GetDuration() >= ticks); + + AudioSegment transmittingAudio; + transmittingAudio.AppendSlice(*audio, previousEnd, GetEnd()); + MOZ_ASSERT(ticks >= transmittingAudio.GetDuration()); + if (transmittingAudio.IsNull()) { + AudioChunk chunk; + chunk.SetNull(ticks); + Unused << mReceiver->EnqueueAudio(chunk); + } else { + for (AudioSegment::ChunkIterator iter(transmittingAudio); !iter.IsEnded(); + iter.Next()) { + Unused << mReceiver->EnqueueAudio(*iter); + } + } +} + +void CrossGraphTransmitter::Destroy() { + MOZ_ASSERT(NS_IsMainThread()); + MediaTrack::Destroy(); + mReceiver->Destroy(); +} + +/** CrossGraphReceiver **/ + +CrossGraphReceiver::CrossGraphReceiver(TrackRate aSampleRate, + TrackRate aTransmitterRate) + : ProcessedMediaTrack(aSampleRate, MediaSegment::AUDIO, + static_cast<MediaSegment*>(new AudioSegment())), + mDriftCorrection(aTransmitterRate, aSampleRate) {} + +void CrossGraphReceiver::ProcessInput(GraphTime aFrom, GraphTime aTo, + uint32_t aFlags) { + LOG(LogLevel::Verbose, + ("Receiver (%p) mSegment: duration: %" PRId64 ", from %" PRId64 + ", to %" PRId64 ", ticks %" PRId64 "", + this, mSegment->GetDuration(), aFrom, aTo, aTo - aFrom)); + + AudioSegment transmittedAudio; + while (mCrossThreadFIFO.AvailableRead()) { + AudioChunk chunk; + Unused << mCrossThreadFIFO.Dequeue(&chunk, 1); + transmittedAudio.AppendAndConsumeChunk(&chunk); + mTransmitterHasStarted = true; + } + + if (mTransmitterHasStarted) { + // If it does not have enough frames the result will be silence. + AudioSegment audioCorrected = + mDriftCorrection.RequestFrames(transmittedAudio, aTo - aFrom); + if (LOG_TEST(LogLevel::Verbose) && audioCorrected.IsNull()) { + LOG(LogLevel::Verbose, + ("Receiver(%p): Silence has been added, not enough input", this)); + } + mSegment->AppendFrom(&audioCorrected); + } else { + mSegment->AppendNullData(aTo - aFrom); + } +} + +int CrossGraphReceiver::EnqueueAudio(AudioChunk& aChunk) { + // This will take place on transmitter graph thread only. + return mCrossThreadFIFO.Enqueue(aChunk); +} + +} // namespace mozilla
new file mode 100644 --- /dev/null +++ b/dom/media/CrossGraphTrack.h @@ -0,0 +1,97 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef MOZILLA_CROSS_GRAPH_TRACK_H_ +#define MOZILLA_CROSS_GRAPH_TRACK_H_ + +#include "AudioDriftCorrection.h" +#include "AudioSegment.h" +#include "ForwardedInputTrack.h" +#include "mozilla/SPSCQueue.h" + +namespace mozilla { +class CrossGraphReceiver; +} + +namespace mozilla::dom { +class AudioStreamTrack; +} + +namespace mozilla { + +/** + * See MediaTrackGraph::CreateCrossGraphTransmitter() + */ +class CrossGraphTransmitter : public ForwardedInputTrack { + friend class CrossGraphManager; + + public: + CrossGraphTransmitter(TrackRate aSampleRate, CrossGraphReceiver* aReceiver); + virtual CrossGraphTransmitter* AsCrossGraphTransmitter() override { + return this; + } + + void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override; + void Destroy() override; + + private: + RefPtr<CrossGraphReceiver> mReceiver; +}; + +/** + * See MediaTrackGraph::CreateCrossGraphReceiver() + */ +class CrossGraphReceiver : public ProcessedMediaTrack { + public: + explicit CrossGraphReceiver(TrackRate aSampleRate, + TrackRate aTransmitterRate); + virtual CrossGraphReceiver* AsCrossGraphReceiver() override { return this; } + + void ProcessInput(GraphTime aFrom, GraphTime aTo, uint32_t aFlags) override; + + int EnqueueAudio(AudioChunk& aChunk); + + private: + SPSCQueue<AudioChunk> mCrossThreadFIFO{30}; + // Indicates that tre CrossGraphTransmitter has started sending frames. It + // is false untill the point, transmitter has sent the first valid frame. + // Accessed in GraphThread only. + bool mTransmitterHasStarted = false; + // Correct the drift between transmitter and receiver. Reciever (this class) + // is considered as the master clock. + // Accessed in GraphThread only. + AudioDriftCorrection mDriftCorrection; +}; + +class CrossGraphManager final { + public: + static CrossGraphManager* Connect( + const RefPtr<dom::AudioStreamTrack>& aStreamTrack, + MediaTrackGraph* aPartnerGraph); + static CrossGraphManager* Connect( + const RefPtr<dom::AudioStreamTrack>& aStreamTrack, AudioDeviceInfo* aSink, + nsPIDOMWindowInner* aWindow); + ~CrossGraphManager() = default; + + void AddAudioOutput(void* aKey); + void RemoveAudioOutput(void* aKey); + void SetAudioOutputVolume(void* aKey, float aVolume); + void Destroy(); + + RefPtr<GenericPromise> EnsureConnected(); + + private: + explicit CrossGraphManager(const RefPtr<MediaInputPort>& aPort) + : mSourcePort(aPort) {} + RefPtr<CrossGraphTransmitter> GetTransmitter(); + RefPtr<CrossGraphReceiver> GetReceiver(); + + // The port that connect the transmitter with the source track. + RefPtr<MediaInputPort> mSourcePort; +}; + +} // namespace mozilla + +#endif /* MOZILLA_CROSS_GRAPH_TRACK_H_ */
--- a/dom/media/ForwardedInputTrack.cpp +++ b/dom/media/ForwardedInputTrack.cpp @@ -9,17 +9,16 @@ #include "AudioNodeEngine.h" #include "AudioNodeExternalInputTrack.h" #include "AudioNodeTrack.h" #include "AudioSegment.h" #include "DOMMediaStream.h" #include "GeckoProfiler.h" #include "ImageContainer.h" #include "MediaTrackGraphImpl.h" -#include "MediaTrackListener.h" #include "mozilla/Attributes.h" #include "mozilla/Logging.h" #include "mozilla/MathAlgorithms.h" #include "mozilla/Unused.h" #include "nsContentUtils.h" #include "nsPrintfCString.h" #include "nsServiceManagerUtils.h" #include "nsWidgetsCID.h"
--- a/dom/media/ForwardedInputTrack.h +++ b/dom/media/ForwardedInputTrack.h @@ -2,16 +2,17 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ #ifndef MOZILLA_FORWARDEDINPUTTRACK_H_ #define MOZILLA_FORWARDEDINPUTTRACK_H_ #include "MediaTrackGraph.h" +#include "MediaTrackListener.h" #include <algorithm> namespace mozilla { /** * See MediaTrackGraph::CreateForwardedInputTrack. */ class ForwardedInputTrack : public ProcessedMediaTrack {
--- a/dom/media/GVAutoplayPermissionRequest.cpp +++ b/dom/media/GVAutoplayPermissionRequest.cpp @@ -2,16 +2,17 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "GVAutoplayPermissionRequest.h" #include "mozilla/dom/HTMLMediaElement.h" #include "mozilla/Logging.h" #include "mozilla/StaticPrefs_media.h" +#include "nsGlobalWindowInner.h" mozilla::LazyLogModule gGVAutoplayRequestLog("GVAutoplay"); namespace mozilla { namespace dom { using RType = GVAutoplayRequestType; using RStatus = GVAutoplayRequestStatus;
--- a/dom/media/MediaTrackGraph.cpp +++ b/dom/media/MediaTrackGraph.cpp @@ -3,16 +3,17 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ #include "MediaTrackGraphImpl.h" #include "mozilla/MathAlgorithms.h" #include "mozilla/Unused.h" #include "AudioSegment.h" +#include "CrossGraphTrack.h" #include "VideoSegment.h" #include "nsContentUtils.h" #include "nsPrintfCString.h" #include "nsServiceManagerUtils.h" #include "prerror.h" #include "mozilla/Logging.h" #include "mozilla/Attributes.h" #include "ForwardedInputTrack.h" @@ -3304,16 +3305,32 @@ ProcessedMediaTrack* MediaTrackGraph::Cr } AudioCaptureTrack* MediaTrackGraph::CreateAudioCaptureTrack() { AudioCaptureTrack* track = new AudioCaptureTrack(GraphRate()); AddTrack(track); return track; } +CrossGraphTransmitter* MediaTrackGraph::CreateCrossGraphTransmitter( + CrossGraphReceiver* aReceiver) { + CrossGraphTransmitter* track = + new CrossGraphTransmitter(GraphRate(), aReceiver); + AddTrack(track); + return track; +} + +CrossGraphReceiver* MediaTrackGraph::CreateCrossGraphReceiver( + TrackRate aTransmitterRate) { + CrossGraphReceiver* track = + new CrossGraphReceiver(GraphRate(), aTransmitterRate); + AddTrack(track); + return track; +} + void MediaTrackGraph::AddTrack(MediaTrack* aTrack) { MediaTrackGraphImpl* graph = static_cast<MediaTrackGraphImpl*>(this); #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED if (graph->mRealtime) { bool found = false; for (auto iter = gGraphs.ConstIter(); !iter.Done(); iter.Next()) { if (iter.UserData() == graph) { found = true;
--- a/dom/media/MediaTrackGraph.h +++ b/dom/media/MediaTrackGraph.h @@ -22,16 +22,18 @@ class nsIRunnable; class nsIGlobalObject; class nsPIDOMWindowInner; namespace mozilla { class AsyncLogger; class AudioCaptureTrack; +class CrossGraphTransmitter; +class CrossGraphReceiver; }; // namespace mozilla extern mozilla::AsyncLogger gMTGTraceLogger; template <> class nsAutoRefTraits<SpeexResamplerState> : public nsPointerRefTraits<SpeexResamplerState> { public: @@ -377,16 +379,18 @@ class MediaTrack : public mozilla::Linke friend class MediaTrackGraphImpl; friend class MediaInputPort; friend class AudioNodeExternalInputTrack; virtual SourceMediaTrack* AsSourceTrack() { return nullptr; } virtual ProcessedMediaTrack* AsProcessedTrack() { return nullptr; } virtual AudioNodeTrack* AsAudioNodeTrack() { return nullptr; } virtual ForwardedInputTrack* AsForwardedInputTrack() { return nullptr; } + virtual CrossGraphTransmitter* AsCrossGraphTransmitter() { return nullptr; } + virtual CrossGraphReceiver* AsCrossGraphReceiver() { return nullptr; } // These Impl methods perform the core functionality of the control methods // above, on the media graph thread. /** * Stop all track activity and disconnect it from all inputs and outputs. * This must be idempotent. */ virtual void DestroyImpl(); @@ -1060,16 +1064,20 @@ class MediaTrackGraph { * for audio or the last video frame for video. */ ProcessedMediaTrack* CreateForwardedInputTrack(MediaSegment::Type aType); /** * Create a track that will mix all its audio inputs. */ AudioCaptureTrack* CreateAudioCaptureTrack(); + CrossGraphTransmitter* CreateCrossGraphTransmitter( + CrossGraphReceiver* aReceiver); + CrossGraphReceiver* CreateCrossGraphReceiver(TrackRate aTransmitterRate); + /** * Add a new track to the graph. Main thread. */ void AddTrack(MediaTrack* aTrack); /* From the main thread, ask the MTG to resolve the returned promise when * the device has started. * The promise is rejected with NS_ERROR_NOT_AVAILABLE if aTrack
--- a/dom/media/gtest/TestAudioDriftCorrection.cpp +++ b/dom/media/gtest/TestAudioDriftCorrection.cpp @@ -325,8 +325,34 @@ void testMonoToStereoInput(int aSourceRa } TEST(TestAudioDriftCorrection, MonoToStereoInput) { testMonoToStereoInput(48000, 48000); testMonoToStereoInput(48000, 44100); testMonoToStereoInput(44100, 48000); } + +TEST(TestAudioDriftCorrection, NotEnoughFrames) +{ + const int32_t sampleRateTransmitter = 48000; + const int32_t sampleRateReceiver = 48000; + AudioDriftCorrection ad(sampleRateTransmitter, sampleRateReceiver); + const int32_t targetFrames = sampleRateReceiver / 100; + + for (int i = 0; i < 7; ++i) { + // Input is something small, 10 frames here, in order to dry out fast, + // after 4 iterations + AudioChunk chunk = CreateAudioChunk<float>(10, 1, AUDIO_FORMAT_FLOAT32); + AudioSegment inSegment; + inSegment.AppendAndConsumeChunk(&chunk); + + AudioSegment outSegment = ad.RequestFrames(inSegment, targetFrames); + EXPECT_EQ(outSegment.GetDuration(), targetFrames); + if (i < 5) { + EXPECT_FALSE(outSegment.IsNull()); + } else { + // Last 2 iterations, the 5th and 6th, will be null. It has used all + // buffered data so the output is silence. + EXPECT_TRUE(outSegment.IsNull()); + } + } +}
--- a/dom/media/moz.build +++ b/dom/media/moz.build @@ -121,16 +121,17 @@ EXPORTS += [ 'BackgroundVideoDecodingPermissionObserver.h', 'Benchmark.h', 'BitReader.h', 'BitWriter.h', 'BufferMediaResource.h', 'BufferReader.h', 'ByteWriter.h', 'ChannelMediaDecoder.h', + 'CrossGraphTrack.h', 'CubebUtils.h', 'DecoderTraits.h', 'DOMMediaStream.h', 'DriftCompensation.h', 'DynamicResampler.h', 'FileBlockCache.h', 'ForwardedInputTrack.h', 'FrameStatistics.h', @@ -249,16 +250,17 @@ UNIFIED_SOURCES += [ 'BaseMediaResource.cpp', 'Benchmark.cpp', 'BitReader.cpp', 'BitWriter.cpp', 'CanvasCaptureMediaStream.cpp', 'ChannelMediaDecoder.cpp', 'ChannelMediaResource.cpp', 'CloneableWithRangeMediaResource.cpp', + 'CrossGraphTrack.cpp', 'DOMMediaStream.cpp', 'DynamicResampler.cpp', 'FileBlockCache.cpp', 'FileMediaResource.cpp', 'ForwardedInputTrack.cpp', 'GetUserMediaRequest.cpp', 'GraphDriver.cpp', 'GraphRunner.cpp',