Bug 1409727 - Add a mode and pref to disallow frame drops in MediaRecorder. r=SingingTree
authorAndreas Pehrson <pehrsons@mozilla.com>
Thu, 19 Oct 2017 14:38:07 +0200
changeset 389828 baf693d2380daa0b704383e890281b1a0d81a575
parent 389827 ea9eada6667e603021ada2e5e63005d6d110d0f5
child 389829 db0fa983b90f44be14aa9b31257149aee28ea6b9
push id54603
push userpehrsons@gmail.com
push dateThu, 02 Nov 2017 10:31:10 +0000
treeherderautoland@baf693d2380d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersSingingTree
bugs1409727
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1409727 - Add a mode and pref to disallow frame drops in MediaRecorder. r=SingingTree MozReview-Commit-ID: CJBJURDn7gB
dom/media/encoder/MediaEncoder.cpp
dom/media/encoder/TrackEncoder.h
dom/media/encoder/VP8TrackEncoder.cpp
dom/media/encoder/VP8TrackEncoder.h
dom/media/gtest/TestVideoTrackEncoder.cpp
dom/media/gtest/TestWebMWriter.cpp
modules/libpref/init/all.js
--- a/dom/media/encoder/MediaEncoder.cpp
+++ b/dom/media/encoder/MediaEncoder.cpp
@@ -627,17 +627,21 @@ MediaEncoder::CreateEncoder(TaskQueue* a
   else if (MediaEncoder::IsWebMEncoderEnabled() &&
       (aMIMEType.EqualsLiteral(VIDEO_WEBM) ||
        (aTrackTypes & ContainerWriter::CREATE_VIDEO_TRACK))) {
     if (aTrackTypes & ContainerWriter::CREATE_AUDIO_TRACK &&
         MediaDecoder::IsOpusEnabled()) {
       audioEncoder = MakeAndAddRef<OpusTrackEncoder>(aTrackRate);
       NS_ENSURE_TRUE(audioEncoder, nullptr);
     }
-    videoEncoder = MakeAndAddRef<VP8TrackEncoder>(aTrackRate);
+    if (Preferences::GetBool("media.recorder.video.frame_drops", true)) {
+      videoEncoder = MakeAndAddRef<VP8TrackEncoder>(aTrackRate, FrameDroppingMode::ALLOW);
+    } else {
+      videoEncoder = MakeAndAddRef<VP8TrackEncoder>(aTrackRate, FrameDroppingMode::DISALLOW);
+    }
     writer = MakeUnique<WebMWriter>(aTrackTypes);
     NS_ENSURE_TRUE(writer, nullptr);
     NS_ENSURE_TRUE(videoEncoder, nullptr);
     mimeType = NS_LITERAL_STRING(VIDEO_WEBM);
   }
 #endif //MOZ_WEBM_ENCODER
   else if (MediaDecoder::IsOggEnabled() && MediaDecoder::IsOpusEnabled() &&
            (aMIMEType.EqualsLiteral(AUDIO_OGG) ||
--- a/dom/media/encoder/TrackEncoder.h
+++ b/dom/media/encoder/TrackEncoder.h
@@ -380,27 +380,33 @@ protected:
   uint32_t mAudioBitrate;
 
   // This may only be accessed on the MSG thread.
   // I.e., in the regular NotifyQueuedChanges for audio to avoid adding data
   // from that callback when the direct one is active.
   bool mDirectConnected;
 };
 
+enum class FrameDroppingMode {
+  ALLOW, // Allowed to drop frames to keep up under load
+  DISALLOW, // Must not drop any frames, even if it means we will OOM
+};
+
 class VideoTrackEncoder : public TrackEncoder
 {
 public:
-  explicit VideoTrackEncoder(TrackRate aTrackRate)
+  VideoTrackEncoder(TrackRate aTrackRate, FrameDroppingMode aFrameDroppingMode)
     : TrackEncoder(aTrackRate)
     , mFrameWidth(0)
     , mFrameHeight(0)
     , mDisplayWidth(0)
     , mDisplayHeight(0)
     , mEncodedTicks(0)
     , mVideoBitrate(0)
+    , mFrameDroppingMode(aFrameDroppingMode)
   {
     mLastChunk.mDuration = 0;
   }
 
   /**
    * Suspends encoding from aTime, i.e., all video frame with a timestamp
    * between aTime and the timestamp of the next Resume() will be dropped.
    */
@@ -546,13 +552,19 @@ protected:
 
   /**
    * The time Suspend was called on the MediaRecorder, so we can calculate the
    * duration on the next Resume().
    */
   TimeStamp mSuspendTime;
 
   uint32_t mVideoBitrate;
+
+  /**
+   * ALLOW to drop frames under load.
+   * DISALLOW to encode all frames, mainly for testing.
+   */
+  FrameDroppingMode mFrameDroppingMode;
 };
 
 } // namespace mozilla
 
 #endif
--- a/dom/media/encoder/VP8TrackEncoder.cpp
+++ b/dom/media/encoder/VP8TrackEncoder.cpp
@@ -51,18 +51,19 @@ GetSourceSurface(already_AddRefed<Image>
     surf = img->GetAsSourceSurface();
     return NS_OK;
   });
 
   NS_DispatchToMainThread(runnable, NS_DISPATCH_SYNC);
   return surf.forget();
 }
 
-VP8TrackEncoder::VP8TrackEncoder(TrackRate aTrackRate)
-  : VideoTrackEncoder(aTrackRate)
+VP8TrackEncoder::VP8TrackEncoder(TrackRate aTrackRate,
+                                 FrameDroppingMode aFrameDroppingMode)
+  : VideoTrackEncoder(aTrackRate, aFrameDroppingMode)
   , mEncodedTimestamp(0)
   , mVPXContext(new vpx_codec_ctx_t())
   , mVPXImageWrapper(new vpx_image_t())
 {
   MOZ_COUNT_CTOR(VP8TrackEncoder);
 }
 
 VP8TrackEncoder::~VP8TrackEncoder()
@@ -571,16 +572,20 @@ nsresult VP8TrackEncoder::PrepareRawFram
  * Compares the elapsed time from the beginning of GetEncodedTrack and
  * the processed frame duration in mSourceSegment
  * in order to set the nextEncodeOperation for next target frame.
  */
 VP8TrackEncoder::EncodeOperation
 VP8TrackEncoder::GetNextEncodeOperation(TimeDuration aTimeElapsed,
                                         StreamTime aProcessedDuration)
 {
+  if (mFrameDroppingMode == FrameDroppingMode::DISALLOW) {
+    return ENCODE_NORMAL_FRAME;
+  }
+
   int64_t durationInUsec =
     FramesToUsecs(aProcessedDuration, mTrackRate).value();
   if (aTimeElapsed.ToMicroseconds() > (durationInUsec * SKIP_FRAME_RATIO)) {
     // The encoder is too slow.
     // We should skip next frame to consume the mSourceSegment.
     return SKIP_FRAME;
   } else if (aTimeElapsed.ToMicroseconds() > (durationInUsec * I_FRAME_RATIO)) {
     // The encoder is a little slow.
--- a/dom/media/encoder/VP8TrackEncoder.h
+++ b/dom/media/encoder/VP8TrackEncoder.h
@@ -22,18 +22,19 @@ typedef struct vpx_image vpx_image_t;
  */
 class VP8TrackEncoder : public VideoTrackEncoder
 {
   enum EncodeOperation {
     ENCODE_NORMAL_FRAME, // VP8 track encoder works normally.
     ENCODE_I_FRAME, // The next frame will be encoded as I-Frame.
     SKIP_FRAME, // Skip the next frame.
   };
+
 public:
-  explicit VP8TrackEncoder(TrackRate aTrackRate);
+  VP8TrackEncoder(TrackRate aTrackRate, FrameDroppingMode aFrameDroppingMode);
   virtual ~VP8TrackEncoder();
 
   already_AddRefed<TrackMetadataBase> GetMetadata() final override;
 
   nsresult GetEncodedTrack(EncodedFrameContainer& aData) final override;
 
 protected:
   nsresult Init(int32_t aWidth, int32_t aHeight,
--- a/dom/media/gtest/TestVideoTrackEncoder.cpp
+++ b/dom/media/gtest/TestVideoTrackEncoder.cpp
@@ -187,17 +187,17 @@ struct InitParam {
   int  mWidth;          // frame width
   int  mHeight;         // frame height
 };
 
 class TestVP8TrackEncoder: public VP8TrackEncoder
 {
 public:
   explicit TestVP8TrackEncoder(TrackRate aTrackRate = VIDEO_TRACK_RATE)
-    : VP8TrackEncoder(aTrackRate) {}
+    : VP8TrackEncoder(aTrackRate, FrameDroppingMode::DISALLOW) {}
 
   ::testing::AssertionResult TestInit(const InitParam &aParam)
   {
     nsresult result = Init(aParam.mWidth, aParam.mHeight, aParam.mWidth, aParam.mHeight);
 
     if (((NS_FAILED(result) && aParam.mShouldSucceed)) || (NS_SUCCEEDED(result) && !aParam.mShouldSucceed))
     {
       return ::testing::AssertionFailure()
--- a/dom/media/gtest/TestWebMWriter.cpp
+++ b/dom/media/gtest/TestWebMWriter.cpp
@@ -26,17 +26,17 @@ public:
     return false;
   }
 };
 
 class WebMVP8TrackEncoder: public VP8TrackEncoder
 {
 public:
   explicit WebMVP8TrackEncoder(TrackRate aTrackRate = 90000)
-    : VP8TrackEncoder(aTrackRate) {}
+    : VP8TrackEncoder(aTrackRate, FrameDroppingMode::DISALLOW) {}
 
   bool TestVP8Creation(int32_t aWidth, int32_t aHeight, int32_t aDisplayWidth,
                        int32_t aDisplayHeight)
   {
     if (NS_SUCCEEDED(Init(aWidth, aHeight, aDisplayWidth, aDisplayHeight))) {
       return true;
     }
     return false;
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -614,16 +614,20 @@ pref("media.webspeech.synth.enabled", fa
 #endif
 #ifdef MOZ_WEBM_ENCODER
 pref("media.encoder.webm.enabled", true);
 #endif
 
 // Whether to allow recording of AudioNodes with MediaRecorder
 pref("media.recorder.audio_node.enabled", false);
 
+// Whether MediaRecorder's video encoder should allow dropping frames in order
+// to keep up under load. Useful for tests but beware of memory consumption!
+pref("media.recorder.video.frame_drops", true);
+
 // Whether to autostart a media element with an |autoplay| attribute
 pref("media.autoplay.enabled", true);
 
 // The default number of decoded video frames that are enqueued in
 // MediaDecoderReader's mVideoQueue.
 pref("media.video-queue.default-size", 10);
 
 // The maximum number of queued frames to send to the compositor.