Bug 1423253 - Add a drift compensator to reclock video timestamps. r=padenot
authorAndreas Pehrson <apehrson@mozilla.com>
Fri, 22 Mar 2019 11:42:31 +0000
changeset 465636 60d215858649ce5f058db2f715004749c17e7c93
parent 465635 54b4c9399b81821c28b0f6cad7be8676575c9f24
child 465637 9478255cfe12901e4f161fa35e6696147cdb110b
push id35744
push userapavel@mozilla.com
push dateFri, 22 Mar 2019 16:44:08 +0000
treeherdermozilla-central@e66a2b59914d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerspadenot
bugs1423253
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1423253 - Add a drift compensator to reclock video timestamps. r=padenot Differential Revision: https://phabricator.services.mozilla.com/D22902
dom/media/DriftCompensation.h
dom/media/gtest/TestDriftCompensation.cpp
dom/media/gtest/moz.build
dom/media/moz.build
new file mode 100644
--- /dev/null
+++ b/dom/media/DriftCompensation.h
@@ -0,0 +1,136 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+#ifndef DriftCompensation_h_
+#define DriftCompensation_h_
+
+#include "MediaSegment.h"
+#include "VideoUtils.h"
+#include "mozilla/Atomics.h"
+#include "mozilla/Unused.h"
+
+namespace mozilla {
+
+static LazyLogModule gDriftCompensatorLog("DriftCompensator");
+#define LOG(type, ...) MOZ_LOG(gDriftCompensatorLog, type, (__VA_ARGS__))
+
+/**
+ * DriftCompensator can be used to handle drift between audio and video tracks
+ * from the MediaStreamGraph.
+ *
+ * Drift can occur because audio is driven by a MediaStreamGraph running off an
+ * audio callback, thus it's progressed by the clock of one the audio output
+ * devices on the user's machine. Video on the other hand is always expressed in
+ * wall-clock TimeStamps, i.e., it's progressed by the system clock. These
+ * clocks will, over time, drift apart.
+ *
+ * Do not use the DriftCompensator across multiple audio tracks, as it will
+ * automatically record the start time of the first audio samples, and all
+ * samples for the same audio track on the same audio clock will have to be
+ * processed to retain accuracy.
+ *
+ * DriftCompensator is designed to be used from two threads:
+ * - The audio thread for notifications of audio samples.
+ * - The video thread for compensating drift of video frames to match the audio
+ *   clock.
+ */
+class DriftCompensator {
+  const RefPtr<nsIEventTarget> mVideoThread;
+  const TrackRate mAudioRate;
+
+  // Number of audio samples produced. Any thread.
+  Atomic<StreamTime> mAudioSamples{0};
+
+  // Time the first audio samples were added. mVideoThread only.
+  TimeStamp mAudioStartTime;
+
+  void SetAudioStartTime(TimeStamp aTime) {
+    MOZ_ASSERT(mVideoThread->IsOnCurrentThread());
+    MOZ_ASSERT(mAudioStartTime.IsNull());
+    mAudioStartTime = aTime;
+  }
+
+ protected:
+  virtual ~DriftCompensator() = default;
+
+ public:
+  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(DriftCompensator)
+
+  DriftCompensator(RefPtr<nsIEventTarget> aVideoThread, TrackRate aAudioRate)
+      : mVideoThread(std::move(aVideoThread)), mAudioRate(aAudioRate) {
+    MOZ_ASSERT(mAudioRate > 0);
+  }
+
+  void NotifyAudioStart(TimeStamp aStart) {
+    MOZ_ASSERT(mAudioSamples == 0);
+    LOG(LogLevel::Info, "DriftCompensator %p at rate %d started", this,
+        mAudioRate);
+    nsresult rv = mVideoThread->Dispatch(NewRunnableMethod<TimeStamp>(
+        "DriftCompensator::SetAudioStartTime", this,
+        &DriftCompensator::SetAudioStartTime, aStart));
+    MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
+    Unused << rv;
+  }
+
+  /**
+   * aSamples is the number of samples fed by an AudioStream.
+   */
+  void NotifyAudio(StreamTime aSamples) {
+    MOZ_ASSERT(aSamples > 0);
+    mAudioSamples += aSamples;
+
+    LOG(LogLevel::Verbose,
+        "DriftCompensator %p Processed another %" PRId64
+        " samples; now %.3fs audio",
+        this, aSamples, static_cast<double>(mAudioSamples) / mAudioRate);
+  }
+
+  /**
+   * Drift compensates a video TimeStamp based on historical audio data.
+   */
+  virtual TimeStamp GetVideoTime(TimeStamp aNow, TimeStamp aTime) {
+    MOZ_ASSERT(mVideoThread->IsOnCurrentThread());
+    StreamTime samples = mAudioSamples;
+
+    if (samples / mAudioRate < 10) {
+      // We don't apply compensation for the first 10 seconds because of the
+      // higher inaccuracy during this time.
+      LOG(LogLevel::Debug, "DriftCompensator %p %" PRId64 "ms so far; ignoring",
+          this, samples * 1000 / mAudioRate);
+      return aTime;
+    }
+
+    int64_t videoScaleUs = (aNow - mAudioStartTime).ToMicroseconds();
+    int64_t audioScaleUs = FramesToUsecs(samples, mAudioRate).value();
+    int64_t videoDurationUs = (aTime - mAudioStartTime).ToMicroseconds();
+
+    if (videoScaleUs == 0) {
+      videoScaleUs = audioScaleUs;
+    }
+
+    TimeStamp reclocked =
+        mAudioStartTime +
+        TimeDuration::FromMicroseconds(
+            SaferMultDiv(videoDurationUs, audioScaleUs, videoScaleUs).value());
+
+    LOG(LogLevel::Debug,
+        "DriftCompensator %p GetVideoTime, v-now: %.3fs, a-now: %.3fs; %.3fs "
+        "-> %.3fs (d %.3fms)",
+        this, (aNow - mAudioStartTime).ToSeconds(),
+        static_cast<double>(audioScaleUs) / 1000000.0,
+        (aTime - mAudioStartTime).ToSeconds(),
+        (reclocked - mAudioStartTime).ToSeconds(),
+        (reclocked - aTime).ToMilliseconds());
+
+    return reclocked;
+  }
+};
+
+#undef LOG
+
+}  // namespace mozilla
+
+#endif /* DriftCompensation_h_ */
new file mode 100644
--- /dev/null
+++ b/dom/media/gtest/TestDriftCompensation.cpp
@@ -0,0 +1,84 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+
+#include "gtest/gtest.h"
+#include "DriftCompensation.h"
+
+using namespace mozilla;
+
+class DriftCompensatorTest : public ::testing::Test {
+ public:
+  const TrackRate mRate = 44100;
+  const TimeStamp mStart;
+  const RefPtr<DriftCompensator> mComp;
+
+  DriftCompensatorTest()
+      : mStart(TimeStamp::Now()),
+        mComp(MakeRefPtr<DriftCompensator>(GetCurrentThreadEventTarget(),
+                                           mRate)) {
+    mComp->NotifyAudioStart(mStart);
+    // NotifyAudioStart dispatched a runnable to update the audio mStart time on
+    // the video thread. Because this is a test, the video thread is the current
+    // thread. We spin the event loop until we know the mStart time is updated.
+    {
+      bool updated = false;
+      NS_DispatchToCurrentThread(
+          NS_NewRunnableFunction(__func__, [&] { updated = true; }));
+      SpinEventLoopUntil([&] { return updated; });
+    }
+  }
+
+  // Past() is half as far from `mStart` as `aNow`.
+  TimeStamp Past(TimeStamp aNow) {
+    return mStart + (aNow - mStart) / (int64_t)2;
+  }
+
+  // Future() is twice as far from `mStart` as `aNow`.
+  TimeStamp Future(TimeStamp aNow) { return mStart + (aNow - mStart) * 2; }
+};
+
+TEST_F(DriftCompensatorTest, Initialized) {
+  EXPECT_EQ(mComp->GetVideoTime(mStart, mStart), mStart);
+}
+
+TEST_F(DriftCompensatorTest, SlowerAudio) {
+  // 10s of audio took 20 seconds of wall clock to play out
+  mComp->NotifyAudio(mRate * 10);
+  TimeStamp now = mStart + TimeDuration::FromSeconds(20);
+  EXPECT_EQ((mComp->GetVideoTime(now, mStart) - mStart).ToSeconds(), 0.0);
+  EXPECT_EQ((mComp->GetVideoTime(now, Past(now)) - mStart).ToSeconds(), 5.0);
+  EXPECT_EQ((mComp->GetVideoTime(now, now) - mStart).ToSeconds(), 10.0);
+  EXPECT_EQ((mComp->GetVideoTime(now, Future(now)) - mStart).ToSeconds(), 20.0);
+}
+
+TEST_F(DriftCompensatorTest, NoDrift) {
+  // 10s of audio took 10 seconds of wall clock to play out
+  mComp->NotifyAudio(mRate * 10);
+  TimeStamp now = mStart + TimeDuration::FromSeconds(10);
+  EXPECT_EQ((mComp->GetVideoTime(now, mStart) - mStart).ToSeconds(), 0.0);
+  EXPECT_EQ((mComp->GetVideoTime(now, Past(now)) - mStart).ToSeconds(), 5.0);
+  EXPECT_EQ((mComp->GetVideoTime(now, now) - mStart).ToSeconds(), 10.0);
+  EXPECT_EQ((mComp->GetVideoTime(now, Future(now)) - mStart).ToSeconds(), 20.0);
+}
+
+TEST_F(DriftCompensatorTest, NoProgress) {
+  // 10s of audio took 0 seconds of wall clock to play out
+  mComp->NotifyAudio(mRate * 10);
+  TimeStamp now = mStart;
+  TimeStamp future = mStart + TimeDuration::FromSeconds(5);
+  EXPECT_EQ((mComp->GetVideoTime(now, mStart) - mStart).ToSeconds(), 0.0);
+  EXPECT_EQ((mComp->GetVideoTime(now, future) - mStart).ToSeconds(), 5.0);
+}
+
+TEST_F(DriftCompensatorTest, FasterAudio) {
+  // 20s of audio took 10 seconds of wall clock to play out
+  mComp->NotifyAudio(mRate * 20);
+  TimeStamp now = mStart + TimeDuration::FromSeconds(10);
+  EXPECT_EQ((mComp->GetVideoTime(now, mStart) - mStart).ToSeconds(), 0.0);
+  EXPECT_EQ((mComp->GetVideoTime(now, Past(now)) - mStart).ToSeconds(), 10.0);
+  EXPECT_EQ((mComp->GetVideoTime(now, now) - mStart).ToSeconds(), 20.0);
+  EXPECT_EQ((mComp->GetVideoTime(now, Future(now)) - mStart).ToSeconds(), 40.0);
+}
--- a/dom/media/gtest/moz.build
+++ b/dom/media/gtest/moz.build
@@ -20,16 +20,17 @@ UNIFIED_SOURCES += [
     'TestAudioMixer.cpp',
     'TestAudioPacketizer.cpp',
     'TestAudioSegment.cpp',
     'TestAudioTrackEncoder.cpp',
     'TestBitWriter.cpp',
     'TestBlankVideoDataCreator.cpp',
     'TestCDMStorage.cpp',
     'TestDataMutex.cpp',
+    'TestDriftCompensation.cpp',
     'TestGMPCrossOrigin.cpp',
     'TestGMPRemoveAndDelete.cpp',
     'TestGMPUtils.cpp',
     'TestGroupId.cpp',
     'TestIntervalSet.cpp',
     'TestMediaDataDecoder.cpp',
     'TestMediaDataEncoder.cpp',
     'TestMediaEventSource.cpp',
--- a/dom/media/moz.build
+++ b/dom/media/moz.build
@@ -110,16 +110,17 @@ EXPORTS += [
     'BitWriter.h',
     'BufferMediaResource.h',
     'BufferReader.h',
     'ByteWriter.h',
     'ChannelMediaDecoder.h',
     'CubebUtils.h',
     'DecoderTraits.h',
     'DOMMediaStream.h',
+    'DriftCompensation.h',
     'FileBlockCache.h',
     'FrameStatistics.h',
     'ImageToI420.h',
     'Intervals.h',
     'MediaCache.h',
     'MediaContainerType.h',
     'MediaData.h',
     'MediaDataDemuxer.h',