Bug 1423253 - Add drift compensation to MediaEncoder. r=padenot
authorAndreas Pehrson <apehrson@mozilla.com>
Fri, 22 Mar 2019 11:42:38 +0000
changeset 465637 9478255cfe12901e4f161fa35e6696147cdb110b
parent 465636 60d215858649ce5f058db2f715004749c17e7c93
child 465638 3de6381be3d90f8b1a864ef0ebad8d47f87da36d
push id35744
push userapavel@mozilla.com
push dateFri, 22 Mar 2019 16:44:08 +0000
treeherdermozilla-central@e66a2b59914d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerspadenot
bugs1423253
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1423253 - Add drift compensation to MediaEncoder. r=padenot This plumbs the DriftCompensator into the AudioTrackListener and VideoTrackEncoder. The VideoTrackEncoder is however only finally integrated in the future patch "Disregard VideoChunk durations in VideoTrackEncoder". Differential Revision: https://phabricator.services.mozilla.com/D22903
dom/media/encoder/MediaEncoder.cpp
dom/media/encoder/MediaEncoder.h
dom/media/encoder/TrackEncoder.cpp
dom/media/encoder/TrackEncoder.h
dom/media/encoder/VP8TrackEncoder.cpp
dom/media/encoder/VP8TrackEncoder.h
dom/media/gtest/TestVideoTrackEncoder.cpp
dom/media/gtest/TestWebMWriter.cpp
--- a/dom/media/encoder/MediaEncoder.cpp
+++ b/dom/media/encoder/MediaEncoder.cpp
@@ -3,16 +3,17 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "MediaEncoder.h"
 
 #include <algorithm>
 #include "AudioNodeEngine.h"
 #include "AudioNodeStream.h"
+#include "DriftCompensation.h"
 #include "GeckoProfiler.h"
 #include "MediaDecoder.h"
 #include "MediaStreamListener.h"
 #include "mozilla/dom/AudioNode.h"
 #include "mozilla/dom/AudioStreamTrack.h"
 #include "mozilla/dom/MediaStreamTrack.h"
 #include "mozilla/dom/VideoStreamTrack.h"
 #include "mozilla/gfx/Point.h"  // IntSize
@@ -44,20 +45,22 @@ mozilla::LazyLogModule gMediaEncoderLog(
 
 namespace mozilla {
 
 using namespace dom;
 using namespace media;
 
 class MediaEncoder::AudioTrackListener : public DirectMediaStreamTrackListener {
  public:
-  AudioTrackListener(AudioTrackEncoder* aEncoder, TaskQueue* aEncoderThread)
+  AudioTrackListener(DriftCompensator* aDriftCompensator,
+                     AudioTrackEncoder* aEncoder, TaskQueue* aEncoderThread)
       : mDirectConnected(false),
         mInitialized(false),
         mRemoved(false),
+        mDriftCompensator(aDriftCompensator),
         mEncoder(aEncoder),
         mEncoderThread(aEncoderThread) {
     MOZ_ASSERT(mEncoder);
     MOZ_ASSERT(mEncoderThread);
   }
 
   void NotifyShutdown() { mShutdown = true; }
 
@@ -86,24 +89,27 @@ class MediaEncoder::AudioTrackListener :
     MOZ_ASSERT(mEncoder);
     MOZ_ASSERT(mEncoderThread);
 
     if (mShutdown) {
       return;
     }
 
     if (!mInitialized) {
+      mDriftCompensator->NotifyAudioStart(TimeStamp::Now());
       nsresult rv = mEncoderThread->Dispatch(NewRunnableMethod<StreamTime>(
           "mozilla::AudioTrackEncoder::SetStartOffset", mEncoder,
           &AudioTrackEncoder::SetStartOffset, aTrackOffset));
       MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
       Unused << rv;
       mInitialized = true;
     }
 
+    mDriftCompensator->NotifyAudio(aQueuedMedia.GetDuration());
+
     if (!mDirectConnected) {
       NotifyRealtimeTrackData(aGraph, aTrackOffset, aQueuedMedia);
     }
 
     AutoTArray<Pair<bool, StreamTime>, 2> nulledSequence;
     for (AudioSegment::ConstChunkIterator iter(
              static_cast<const AudioSegment&>(aQueuedMedia));
          !iter.IsEnded(); iter.Next()) {
@@ -192,16 +198,17 @@ class MediaEncoder::AudioTrackListener :
   }
 
  private:
   // True when MediaEncoder has shutdown and destroyed the TaskQueue.
   Atomic<bool> mShutdown;
   bool mDirectConnected;
   bool mInitialized;
   bool mRemoved;
+  const RefPtr<DriftCompensator> mDriftCompensator;
   RefPtr<AudioTrackEncoder> mEncoder;
   RefPtr<TaskQueue> mEncoderThread;
 };
 
 class MediaEncoder::VideoTrackListener : public DirectMediaStreamTrackListener {
  public:
   VideoTrackListener(VideoTrackEncoder* aEncoder, TaskQueue* aEncoderThread)
       : mDirectConnected(false),
@@ -295,17 +302,23 @@ class MediaEncoder::VideoTrackListener :
     MOZ_ASSERT(aMedia.GetType() == MediaSegment::VIDEO);
 
     if (mShutdown) {
       return;
     }
 
     const VideoSegment& video = static_cast<const VideoSegment&>(aMedia);
     VideoSegment copy;
-    copy.AppendSlice(video, 0, video.GetDuration());
+    for (VideoSegment::ConstChunkIterator iter(video); !iter.IsEnded();
+         iter.Next()) {
+      copy.AppendFrame(do_AddRef(iter->mFrame.GetImage()), 1,
+                       iter->mFrame.GetIntrinsicSize(),
+                       iter->mFrame.GetPrincipalHandle(),
+                       iter->mFrame.GetForceBlack(), iter->mTimeStamp);
+    }
 
     nsresult rv = mEncoderThread->Dispatch(
         NewRunnableMethod<StoreCopyPassByRRef<VideoSegment>>(
             "mozilla::VideoTrackEncoder::AppendVideoSegment", mEncoder,
             &VideoTrackEncoder::AppendVideoSegment, std::move(copy)));
     MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
     Unused << rv;
   }
@@ -426,36 +439,37 @@ class MediaEncoder::EncoderListener : pu
 
  protected:
   RefPtr<TaskQueue> mEncoderThread;
   RefPtr<MediaEncoder> mEncoder;
   bool mPendingDataAvailable;
 };
 
 MediaEncoder::MediaEncoder(TaskQueue* aEncoderThread,
+                           RefPtr<DriftCompensator> aDriftCompensator,
                            UniquePtr<ContainerWriter> aWriter,
                            AudioTrackEncoder* aAudioEncoder,
                            VideoTrackEncoder* aVideoEncoder,
-                           const nsAString& aMIMEType)
+                           TrackRate aTrackRate, const nsAString& aMIMEType)
     : mEncoderThread(aEncoderThread),
       mWriter(std::move(aWriter)),
       mAudioEncoder(aAudioEncoder),
       mVideoEncoder(aVideoEncoder),
       mEncoderListener(MakeAndAddRef<EncoderListener>(mEncoderThread, this)),
       mStartTime(TimeStamp::Now()),
       mMIMEType(aMIMEType),
       mInitialized(false),
       mMetadataEncoded(false),
       mCompleted(false),
       mError(false),
       mCanceled(false),
       mShutdown(false) {
   if (mAudioEncoder) {
-    mAudioListener =
-        MakeAndAddRef<AudioTrackListener>(mAudioEncoder, mEncoderThread);
+    mAudioListener = MakeAndAddRef<AudioTrackListener>(
+        aDriftCompensator, mAudioEncoder, mEncoderThread);
     nsresult rv =
         mEncoderThread->Dispatch(NewRunnableMethod<RefPtr<EncoderListener>>(
             "mozilla::AudioTrackEncoder::RegisterListener", mAudioEncoder,
             &AudioTrackEncoder::RegisterListener, mEncoderListener));
     MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
     Unused << rv;
   }
   if (mVideoEncoder) {
@@ -628,16 +642,18 @@ already_AddRefed<MediaEncoder> MediaEnco
     TaskQueue* aEncoderThread, const nsAString& aMIMEType,
     uint32_t aAudioBitrate, uint32_t aVideoBitrate, uint8_t aTrackTypes,
     TrackRate aTrackRate) {
   AUTO_PROFILER_LABEL("MediaEncoder::CreateEncoder", OTHER);
 
   UniquePtr<ContainerWriter> writer;
   RefPtr<AudioTrackEncoder> audioEncoder;
   RefPtr<VideoTrackEncoder> videoEncoder;
+  auto driftCompensator =
+      MakeRefPtr<DriftCompensator>(aEncoderThread, aTrackRate);
   nsString mimeType;
 
   if (!aTrackTypes) {
     MOZ_ASSERT(false);
     LOG(LogLevel::Error, ("No TrackTypes"));
     return nullptr;
   }
 #ifdef MOZ_WEBM_ENCODER
@@ -645,21 +661,21 @@ already_AddRefed<MediaEncoder> MediaEnco
            (aMIMEType.EqualsLiteral(VIDEO_WEBM) ||
             (aTrackTypes & ContainerWriter::CREATE_VIDEO_TRACK))) {
     if (aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK &&
         MediaDecoder::IsOpusEnabled()) {
       audioEncoder = MakeAndAddRef<OpusTrackEncoder>(aTrackRate);
       NS_ENSURE_TRUE(audioEncoder, nullptr);
     }
     if (Preferences::GetBool("media.recorder.video.frame_drops", true)) {
-      videoEncoder =
-          MakeAndAddRef<VP8TrackEncoder>(aTrackRate, FrameDroppingMode::ALLOW);
+      videoEncoder = MakeAndAddRef<VP8TrackEncoder>(
+          driftCompensator, aTrackRate, FrameDroppingMode::ALLOW);
     } else {
       videoEncoder = MakeAndAddRef<VP8TrackEncoder>(
-          aTrackRate, FrameDroppingMode::DISALLOW);
+          driftCompensator, aTrackRate, FrameDroppingMode::DISALLOW);
     }
     writer = MakeUnique<WebMWriter>(aTrackTypes);
     NS_ENSURE_TRUE(writer, nullptr);
     NS_ENSURE_TRUE(videoEncoder, nullptr);
     mimeType = NS_LITERAL_STRING(VIDEO_WEBM);
   }
 #endif  // MOZ_WEBM_ENCODER
   else if (MediaDecoder::IsOggEnabled() && MediaDecoder::IsOpusEnabled() &&
@@ -688,18 +704,19 @@ already_AddRefed<MediaEncoder> MediaEnco
     }
   }
   if (videoEncoder) {
     videoEncoder->SetWorkerThread(aEncoderThread);
     if (aVideoBitrate != 0) {
       videoEncoder->SetBitrate(aVideoBitrate);
     }
   }
-  return MakeAndAddRef<MediaEncoder>(aEncoderThread, std::move(writer),
-                                     audioEncoder, videoEncoder, mimeType);
+  return MakeAndAddRef<MediaEncoder>(
+      aEncoderThread, std::move(driftCompensator), std::move(writer),
+      audioEncoder, videoEncoder, aTrackRate, mimeType);
 }
 
 nsresult MediaEncoder::GetEncodedMetadata(
     nsTArray<nsTArray<uint8_t>>* aOutputBufs, nsAString& aMIMEType) {
   AUTO_PROFILER_LABEL("MediaEncoder::GetEncodedMetadata", OTHER);
 
   MOZ_ASSERT(mEncoderThread->IsCurrentThreadIn());
 
--- a/dom/media/encoder/MediaEncoder.h
+++ b/dom/media/encoder/MediaEncoder.h
@@ -13,16 +13,17 @@
 #include "mozilla/DebugOnly.h"
 #include "mozilla/MemoryReporting.h"
 #include "mozilla/UniquePtr.h"
 #include "nsIMemoryReporter.h"
 #include "TrackEncoder.h"
 
 namespace mozilla {
 
+class DriftCompensator;
 class TaskQueue;
 
 namespace dom {
 class AudioNode;
 class AudioStreamTrack;
 class MediaStreamTrack;
 class VideoStreamTrack;
 }  // namespace dom
@@ -102,19 +103,22 @@ class MediaEncoder {
  private:
   class AudioTrackListener;
   class VideoTrackListener;
   class EncoderListener;
 
  public:
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaEncoder)
 
-  MediaEncoder(TaskQueue* aEncoderThread, UniquePtr<ContainerWriter> aWriter,
+  MediaEncoder(TaskQueue* aEncoderThread,
+               RefPtr<DriftCompensator> aDriftCompensator,
+               UniquePtr<ContainerWriter> aWriter,
                AudioTrackEncoder* aAudioEncoder,
-               VideoTrackEncoder* aVideoEncoder, const nsAString& aMIMEType);
+               VideoTrackEncoder* aVideoEncoder, TrackRate aTrackRate,
+               const nsAString& aMIMEType);
 
   /* Note - called from control code, not on MSG threads. */
   void Suspend(TimeStamp aTime);
 
   /**
    * Note - called from control code, not on MSG threads.
    * Calculates time spent paused in order to offset frames. */
   void Resume(TimeStamp aTime);
--- a/dom/media/encoder/TrackEncoder.cpp
+++ b/dom/media/encoder/TrackEncoder.cpp
@@ -365,19 +365,21 @@ void AudioTrackEncoder::DeInterleaveTrac
 
 size_t AudioTrackEncoder::SizeOfExcludingThis(
     mozilla::MallocSizeOf aMallocSizeOf) {
   MOZ_ASSERT(!mWorkerThread || mWorkerThread->IsCurrentThreadIn());
   return mIncomingBuffer.SizeOfExcludingThis(aMallocSizeOf) +
          mOutgoingBuffer.SizeOfExcludingThis(aMallocSizeOf);
 }
 
-VideoTrackEncoder::VideoTrackEncoder(TrackRate aTrackRate,
+VideoTrackEncoder::VideoTrackEncoder(RefPtr<DriftCompensator> aDriftCompensator,
+                                     TrackRate aTrackRate,
                                      FrameDroppingMode aFrameDroppingMode)
     : TrackEncoder(aTrackRate),
+      mDriftCompensator(std::move(aDriftCompensator)),
       mFrameWidth(0),
       mFrameHeight(0),
       mDisplayWidth(0),
       mDisplayHeight(0),
       mEncodedTicks(0),
       mVideoBitrate(0),
       mFrameDroppingMode(aFrameDroppingMode),
       mKeyFrameInterval(DEFAULT_KEYFRAME_INTERVAL_MS) {
--- a/dom/media/encoder/TrackEncoder.h
+++ b/dom/media/encoder/TrackEncoder.h
@@ -11,16 +11,17 @@
 #include "MediaStreamGraph.h"
 #include "StreamTracks.h"
 #include "TrackMetadataBase.h"
 #include "VideoSegment.h"
 
 namespace mozilla {
 
 class AbstractThread;
+class DriftCompensator;
 class TrackEncoder;
 
 class TrackEncoderListener {
  public:
   NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackEncoderListener)
 
   /**
    * Called when the TrackEncoder's underlying encoder has been successfully
@@ -381,17 +382,18 @@ class AudioTrackEncoder : public TrackEn
 
 enum class FrameDroppingMode {
   ALLOW,     // Allowed to drop frames to keep up under load
   DISALLOW,  // Must not drop any frames, even if it means we will OOM
 };
 
 class VideoTrackEncoder : public TrackEncoder {
  public:
-  explicit VideoTrackEncoder(TrackRate aTrackRate,
+  explicit VideoTrackEncoder(RefPtr<DriftCompensator> aDriftCompensator,
+                             TrackRate aTrackRate,
                              FrameDroppingMode aFrameDroppingMode);
 
   /**
    * Suspends encoding from aTime, i.e., all video frame with a timestamp
    * between aTime and the timestamp of the next Resume() will be dropped.
    */
   void Suspend(TimeStamp aTime) override;
 
@@ -477,16 +479,22 @@ class VideoTrackEncoder : public TrackEn
    * height of source frames, this initialization is delayed until we have
    * received the first valid video frame from MediaStreamGraph.
    * Listeners will be notified after it has been successfully initialized.
    */
   virtual nsresult Init(int aWidth, int aHeight, int aDisplayWidth,
                         int aDisplayHeight) = 0;
 
   /**
+   * Drift compensator for re-clocking incoming video frame wall-clock
+   * timestamps to audio time.
+   */
+  const RefPtr<DriftCompensator> mDriftCompensator;
+
+  /**
    * The width of source video frame, ceiled if the source width is odd.
    */
   int mFrameWidth;
 
   /**
    * The height of source video frame, ceiled if the source height is odd.
    */
   int mFrameHeight;
--- a/dom/media/encoder/VP8TrackEncoder.cpp
+++ b/dom/media/encoder/VP8TrackEncoder.cpp
@@ -28,19 +28,21 @@ LazyLogModule gVP8TrackEncoderLog("VP8Tr
 #define DEFAULT_BITRATE_BPS 2500000
 #define MAX_KEYFRAME_INTERVAL 600
 
 using namespace mozilla::gfx;
 using namespace mozilla::layers;
 using namespace mozilla::media;
 using namespace mozilla::dom;
 
-VP8TrackEncoder::VP8TrackEncoder(TrackRate aTrackRate,
+VP8TrackEncoder::VP8TrackEncoder(RefPtr<DriftCompensator> aDriftCompensator,
+                                 TrackRate aTrackRate,
                                  FrameDroppingMode aFrameDroppingMode)
-    : VideoTrackEncoder(aTrackRate, aFrameDroppingMode),
+    : VideoTrackEncoder(std::move(aDriftCompensator), aTrackRate,
+                        aFrameDroppingMode),
       mVPXContext(new vpx_codec_ctx_t()),
       mVPXImageWrapper(new vpx_image_t()) {
   MOZ_COUNT_CTOR(VP8TrackEncoder);
 }
 
 VP8TrackEncoder::~VP8TrackEncoder() {
   Destroy();
   MOZ_COUNT_DTOR(VP8TrackEncoder);
--- a/dom/media/encoder/VP8TrackEncoder.h
+++ b/dom/media/encoder/VP8TrackEncoder.h
@@ -23,17 +23,18 @@ typedef struct vpx_image vpx_image_t;
 class VP8TrackEncoder : public VideoTrackEncoder {
   enum EncodeOperation {
     ENCODE_NORMAL_FRAME,  // VP8 track encoder works normally.
     ENCODE_I_FRAME,       // The next frame will be encoded as I-Frame.
     SKIP_FRAME,           // Skip the next frame.
   };
 
  public:
-  VP8TrackEncoder(TrackRate aTrackRate, FrameDroppingMode aFrameDroppingMode);
+  VP8TrackEncoder(RefPtr<DriftCompensator> aDriftCompensator,
+                  TrackRate aTrackRate, FrameDroppingMode aFrameDroppingMode);
   virtual ~VP8TrackEncoder();
 
   already_AddRefed<TrackMetadataBase> GetMetadata() final;
 
   nsresult GetEncodedTrack(EncodedFrameContainer& aData) final;
 
  protected:
   nsresult Init(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth,
--- a/dom/media/gtest/TestVideoTrackEncoder.cpp
+++ b/dom/media/gtest/TestVideoTrackEncoder.cpp
@@ -1,25 +1,30 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-#include "gtest/gtest.h"
 #include <algorithm>
 
-#include "prtime.h"
-#include "mozilla/ArrayUtils.h"
-#include "VP8TrackEncoder.h"
+#include "DriftCompensation.h"
 #include "ImageContainer.h"
 #include "MediaStreamGraph.h"
 #include "MediaStreamListener.h"
+#include "VP8TrackEncoder.h"
 #include "WebMWriter.h"  // TODO: it's weird to include muxer header to get the class definition of VP8 METADATA
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "mozilla/ArrayUtils.h"
+#include "prtime.h"
 
 #define VIDEO_TRACK_RATE 90000
 
+using ::testing::_;
+using ::testing::Invoke;
+using ::testing::NiceMock;
 using ::testing::TestWithParam;
 using ::testing::Values;
 
 using namespace mozilla::layers;
 using namespace mozilla;
 
 // A helper object to generate of different YUV planes.
 class YUVBufferGenerator {
@@ -174,20 +179,36 @@ class YUVBufferGenerator {
 };
 
 struct InitParam {
   bool mShouldSucceed;  // This parameter should cause success or fail result
   int mWidth;           // frame width
   int mHeight;          // frame height
 };
 
+class MockDriftCompensator : public DriftCompensator {
+ public:
+  MockDriftCompensator()
+      : DriftCompensator(GetCurrentThreadEventTarget(), VIDEO_TRACK_RATE) {
+    ON_CALL(*this, GetVideoTime(_, _))
+        .WillByDefault(Invoke([](TimeStamp, TimeStamp t) { return t; }));
+  }
+
+  MOCK_METHOD2(GetVideoTime, TimeStamp(TimeStamp, TimeStamp));
+};
+
 class TestVP8TrackEncoder : public VP8TrackEncoder {
  public:
   explicit TestVP8TrackEncoder(TrackRate aTrackRate = VIDEO_TRACK_RATE)
-      : VP8TrackEncoder(aTrackRate, FrameDroppingMode::DISALLOW) {}
+      : VP8TrackEncoder(MakeRefPtr<NiceMock<MockDriftCompensator>>(),
+                        aTrackRate, FrameDroppingMode::DISALLOW) {}
+
+  MockDriftCompensator* DriftCompensator() {
+    return static_cast<MockDriftCompensator*>(mDriftCompensator.get());
+  }
 
   ::testing::AssertionResult TestInit(const InitParam& aParam) {
     nsresult result =
         Init(aParam.mWidth, aParam.mHeight, aParam.mWidth, aParam.mHeight);
 
     if (((NS_FAILED(result) && aParam.mShouldSucceed)) ||
         (NS_SUCCEEDED(result) && !aParam.mShouldSucceed)) {
       return ::testing::AssertionFailure()
--- a/dom/media/gtest/TestWebMWriter.cpp
+++ b/dom/media/gtest/TestWebMWriter.cpp
@@ -2,16 +2,17 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "gtest/gtest.h"
 #include "mozilla/CheckedInt.h"
 #include "mozilla/MathAlgorithms.h"
 #include "nestegg/nestegg.h"
+#include "DriftCompensation.h"
 #include "OpusTrackEncoder.h"
 #include "VP8TrackEncoder.h"
 #include "WebMWriter.h"
 
 using namespace mozilla;
 
 class WebMOpusTrackEncoder : public OpusTrackEncoder {
  public:
@@ -23,17 +24,17 @@ class WebMOpusTrackEncoder : public Opus
     }
     return false;
   }
 };
 
 class WebMVP8TrackEncoder : public VP8TrackEncoder {
  public:
   explicit WebMVP8TrackEncoder(TrackRate aTrackRate = 90000)
-      : VP8TrackEncoder(aTrackRate, FrameDroppingMode::DISALLOW) {}
+      : VP8TrackEncoder(nullptr, aTrackRate, FrameDroppingMode::DISALLOW) {}
 
   bool TestVP8Creation(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth,
                        int32_t aDisplayHeight) {
     if (NS_SUCCEEDED(Init(aWidth, aHeight, aDisplayWidth, aDisplayHeight))) {
       return true;
     }
     return false;
   }