Bug 1354457 - Record time spent paused in media encoder and adjust time stamps when muxing. r=pehrsons
authorBryce Van Dyk <bvandyk@mozilla.com>
Fri, 05 May 2017 15:13:11 +1200
changeset 359848 782b0c34f43b6818b2651a209b3f041ea9171241
parent 359847 a17bc9ce68bc88b03100d2144616f3e178d75446
child 359849 3e577500606bd93a80b217d5c7765233d5057995
push id31859
push userihsiao@mozilla.com
push dateMon, 22 May 2017 03:28:26 +0000
treeherdermozilla-central@367944041b55 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerspehrsons
bugs1354457
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1354457 - Record time spent paused in media encoder and adjust time stamps when muxing. r=pehrsons MozReview-Commit-ID: 7J6oQf7qn4u
dom/media/encoder/MediaEncoder.cpp
dom/media/encoder/MediaEncoder.h
--- a/dom/media/encoder/MediaEncoder.cpp
+++ b/dom/media/encoder/MediaEncoder.cpp
@@ -1,16 +1,17 @@
 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 #include "MediaEncoder.h"
 #include "MediaDecoder.h"
 #include "nsIPrincipal.h"
 #include "nsMimeTypes.h"
+#include "TimeUnits.h"
 #include "mozilla/Logging.h"
 #include "mozilla/Preferences.h"
 #include "mozilla/StaticPtr.h"
 #include "mozilla/gfx/Point.h" // IntSize
 
 #include"GeckoProfiler.h"
 #include "OggWriter.h"
 #include "OpusTrackEncoder.h"
@@ -35,16 +36,42 @@ MediaStreamVideoRecorderSink::SetCurrent
   MOZ_ASSERT(mVideoEncoder);
   // If we're suspended (paused) we don't forward frames
   if (!mSuspended) {
     mVideoEncoder->SetCurrentFrames(aSegment);
   }
 }
 
 void
+MediaEncoder::Suspend()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  mLastPauseStartTime = TimeStamp::Now();
+  mSuspended = true;
+  mVideoSink->Suspend();
+}
+
+void
+MediaEncoder::Resume()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  if (!mSuspended) {
+    return;
+  }
+  media::TimeUnit timeSpentPaused =
+    media::TimeUnit::FromTimeDuration(
+      TimeStamp::Now() - mLastPauseStartTime);
+  MOZ_ASSERT(timeSpentPaused.ToMicroseconds() >= 0);
+  MOZ_RELEASE_ASSERT(timeSpentPaused.IsValid());
+  mMicrosecondsSpentPaused += timeSpentPaused.ToMicroseconds();;
+  mSuspended = false;
+  mVideoSink->Resume();
+}
+
+void
 MediaEncoder::SetDirectConnect(bool aConnected)
 {
   mDirectConnected = aConnected;
 }
 
 void
 MediaEncoder::NotifyRealtimeData(MediaStreamGraph* aGraph,
                                  TrackID aID,
@@ -316,16 +343,39 @@ MediaEncoder::WriteEncodedDataToMuxer(Tr
   EncodedFrameContainer encodedVideoData;
   nsresult rv = aTrackEncoder->GetEncodedTrack(encodedVideoData);
   if (NS_FAILED(rv)) {
     // Encoding might be canceled.
     LOG(LogLevel::Error, ("Error! Fail to get encoded data from video encoder."));
     mState = ENCODE_ERROR;
     return rv;
   }
+
+  // Update timestamps to accommodate pauses
+  const nsTArray<RefPtr<EncodedFrame> >& encodedFrames =
+    encodedVideoData.GetEncodedFrames();
+  // Take a copy of the atomic so we don't continually access it
+  uint64_t microsecondsSpentPaused = mMicrosecondsSpentPaused;
+  for (size_t i = 0; i < encodedFrames.Length(); ++i) {
+    RefPtr<EncodedFrame> frame = encodedFrames[i];
+    if (frame->GetTimeStamp() > microsecondsSpentPaused &&
+        frame->GetTimeStamp() - microsecondsSpentPaused > mLastMuxedTimestamp) {
+      // Use the adjusted timestamp if it's after the last timestamp
+      frame->SetTimeStamp(frame->GetTimeStamp() - microsecondsSpentPaused);
+    } else {
+      // If not, we force the last time stamp. We do this so the frames are
+      // still around and in order in case the codec needs to reference them.
+      // Dropping them here may result in artifacts in playback.
+      frame->SetTimeStamp(mLastMuxedTimestamp);
+    }
+    MOZ_ASSERT(mLastMuxedTimestamp <= frame->GetTimeStamp(),
+      "Our frames should be ordered by this point!");
+    mLastMuxedTimestamp = frame->GetTimeStamp();
+  }
+
   rv = mWriter->WriteEncodedTrack(encodedVideoData,
                                   aTrackEncoder->IsEncodingComplete() ?
                                   ContainerWriter::END_OF_STREAM : 0);
   if (NS_FAILED(rv)) {
     LOG(LogLevel::Error, ("Error! Fail to write encoded video track to the media container."));
     mState = ENCODE_ERROR;
   }
   return rv;
--- a/dom/media/encoder/MediaEncoder.h
+++ b/dom/media/encoder/MediaEncoder.h
@@ -98,38 +98,29 @@ public :
     , mVideoSink(new MediaStreamVideoRecorderSink(mVideoEncoder))
     , mStartTime(TimeStamp::Now())
     , mMIMEType(aMIMEType)
     , mSizeOfBuffer(0)
     , mState(MediaEncoder::ENCODE_METADDATA)
     , mShutdown(false)
     , mDirectConnected(false)
     , mSuspended(false)
+    , mMicrosecondsSpentPaused(0)
+    , mLastMuxedTimestamp(0)
 {}
 
   ~MediaEncoder() {};
 
   /* Note - called from control code, not on MSG threads. */
-  void Suspend()
-  {
-    MOZ_ASSERT(NS_IsMainThread());
-    mSuspended = true;
-    mVideoSink->Suspend();
-  }
+  void Suspend();
 
   /**
    * Note - called from control code, not on MSG threads.
-   * Arm to collect the Duration of the next video frame and give it
-   * to the next frame, in order to avoid any possible loss of sync. */
-  void Resume()
-  {
-    MOZ_ASSERT(NS_IsMainThread());
-    mSuspended = false;
-    mVideoSink->Resume();
-  }
+   * Calculates time spent paused in order to offset frames. */
+  void Resume();
 
   /**
    * Tells us which Notify to pay attention to for media
    */
   void SetDirectConnect(bool aConnected);
 
   /**
    * Notified by the AppendToTrack in MediaStreamGraph; aRealtimeMedia is the raw
@@ -239,17 +230,28 @@ private:
   nsAutoPtr<VideoTrackEncoder> mVideoEncoder;
   RefPtr<MediaStreamVideoRecorderSink> mVideoSink;
   TimeStamp mStartTime;
   nsString mMIMEType;
   int64_t mSizeOfBuffer;
   int mState;
   bool mShutdown;
   bool mDirectConnected;
+  // Tracks if the encoder is suspended (paused). Used on the main thread and
+  // MediaRecorder's read thread.
   Atomic<bool> mSuspended;
+  // Timestamp of when the last pause happened. Should only be accessed on the
+  // main thread.
+  TimeStamp mLastPauseStartTime;
+  // Exposes the time spend paused in microseconds. Read by the main thread
+  // and MediaRecorder's read thread. Should only be written by main thread.
+  Atomic<uint64_t> mMicrosecondsSpentPaused;
+  // The timestamp of the last muxed sample. Should only be used on
+  // MediaRecorder's read thread.
+  uint64_t mLastMuxedTimestamp;
   // Get duration from create encoder, for logging purpose
   double GetEncodeTimeStamp()
   {
     TimeDuration decodeTime;
     decodeTime = TimeStamp::Now() - mStartTime;
     return decodeTime.ToMilliseconds();
   }
 };