Bug 1536766 - Handle DecodedStream's resets when calculating VideoChunk durations in the MSG. r=padenot
☠☠ backed out by 41f1dcbe9caa ☠ ☠
authorAndreas Pehrson <apehrson@mozilla.com>
Thu, 18 Apr 2019 15:23:59 +0000
changeset 470099 f9d1f1bfe2b126dc1588e56724b860674519805a
parent 470098 87616997f16017b85f463691262261db6490c30b
child 470100 199efe6aec594ef6d9f3c4c8af16b2e997a8bd51
push id35888
push useraiakab@mozilla.com
push dateFri, 19 Apr 2019 09:47:45 +0000
treeherdermozilla-central@0160424142d1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerspadenot
bugs1536766
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1536766 - Handle DecodedStream's resets when calculating VideoChunk durations in the MSG. r=padenot This is necessary to get an accurate total track duration in case DecodedStream does a lot of seeking and pausing. This also removes previous code for avoiding blocking a video track, as this new code handles that case too. Differential Revision: https://phabricator.services.mozilla.com/D27264
dom/media/MediaStreamGraph.cpp
--- a/dom/media/MediaStreamGraph.cpp
+++ b/dom/media/MediaStreamGraph.cpp
@@ -2513,17 +2513,18 @@ bool SourceMediaStream::PullNewData(Grap
       // (which the track will get as its start time later).
       current = GraphTimeToStreamTime(GraphImpl()->mStateComputedTime);
     } else {
       current = track.mEndOfFlushedData + track.mData->GetDuration();
     }
     if (t <= current) {
       continue;
     }
-    if (!track.mPullingEnabled) {
+    if (!track.mPullingEnabled &&
+        track.mData->GetType() == MediaSegment::AUDIO) {
       if (streamPullingEnabled) {
         LOG(LogLevel::Verbose,
             ("%p: Pulling disabled for track but enabled for stream, append "
              "null data; stream=%p track=%d t=%f current end=%f",
              GraphImpl(), this, track.mID, GraphImpl()->MediaTimeToSeconds(t),
              GraphImpl()->MediaTimeToSeconds(current)));
         track.mData->AppendNullData(t - current);
       }
@@ -2538,16 +2539,77 @@ bool SourceMediaStream::PullNewData(Grap
       if (l.mTrackID == track.mID) {
         l.mListener->NotifyPull(Graph(), current, t);
       }
     }
   }
   return true;
 }
 
+/**
+ * This moves chunks from aIn to aOut. For audio this is simple. For video
+ * we carry durations over if present, or extend up to aDesiredUpToTime if not.
+ *
+ * We also handle "resetters" from captured media elements. This type of source
+ * pushes future frames into the track, and should it need to remove some, e.g.,
+ * because of a seek or pause, it tells us by letting time go backwards. Without
+ * this, tracks would be live for too long after a seek or pause.
+ */
+static void MoveToSegment(SourceMediaStream* aStream, MediaSegment* aIn,
+                          MediaSegment* aOut, StreamTime aCurrentTime,
+                          StreamTime aDesiredUpToTime) {
+  MOZ_ASSERT(aIn->GetType() == aOut->GetType());
+  MOZ_ASSERT(aOut->GetDuration() >= aCurrentTime);
+  if (aIn->GetType() == MediaSegment::AUDIO) {
+    aOut->AppendFrom(aIn);
+  } else {
+    VideoSegment* in = static_cast<VideoSegment*>(aIn);
+    VideoSegment* out = static_cast<VideoSegment*>(aOut);
+    for (VideoSegment::ConstChunkIterator c(*in); !c.IsEnded(); c.Next()) {
+      MOZ_ASSERT(!c->mTimeStamp.IsNull());
+      VideoChunk* last = out->GetLastChunk();
+      if (!last || last->mTimeStamp.IsNull()) {
+        // This is the first frame, or the last frame pushed to `out` has been
+        // all consumed. Just append and we deal with its duration later.
+        out->AppendFrame(do_AddRef(c->mFrame.GetImage()),
+                         c->mFrame.GetIntrinsicSize(),
+                         c->mFrame.GetPrincipalHandle(),
+                         c->mFrame.GetForceBlack(), c->mTimeStamp);
+        if (c->GetDuration() > 0) {
+          out->ExtendLastFrameBy(c->GetDuration());
+        }
+        continue;
+      }
+
+      // We now know when this frame starts, aka when the last frame ends.
+
+      if (c->mTimeStamp < last->mTimeStamp) {
+        // Time is going backwards. This is a resetting frame from
+        // DecodedStream. Clear everything up to currentTime.
+        out->Clear();
+        out->AppendNullData(aCurrentTime);
+      }
+
+      // Append the current frame (will have duration 0).
+      out->AppendFrame(do_AddRef(c->mFrame.GetImage()),
+                       c->mFrame.GetIntrinsicSize(),
+                       c->mFrame.GetPrincipalHandle(),
+                       c->mFrame.GetForceBlack(), c->mTimeStamp);
+      if (c->GetDuration() > 0) {
+        out->ExtendLastFrameBy(c->GetDuration());
+      }
+    }
+    if (out->GetDuration() < aDesiredUpToTime) {
+      out->ExtendLastFrameBy(aDesiredUpToTime - out->GetDuration());
+    }
+    in->Clear();
+  }
+  MOZ_ASSERT(aIn->GetDuration() == 0, "aIn must be consumed");
+}
+
 void SourceMediaStream::ExtractPendingInput(GraphTime aCurrentTime,
                                             GraphTime aDesiredUpToTime) {
   MutexAutoLock lock(mMutex);
 
   bool finished = mFinishPending;
   StreamTime streamCurrentTime = GraphTimeToStreamTime(aCurrentTime);
   StreamTime streamDesiredUpToTime = GraphTimeToStreamTime(aDesiredUpToTime);
 
@@ -2565,53 +2627,44 @@ void SourceMediaStream::ExtractPendingIn
 
     for (TrackBound<MediaStreamTrackListener>& b : mTrackListeners) {
       if (b.mTrackID != data->mID) {
         continue;
       }
       b.mListener->NotifyQueuedChanges(GraphImpl(), offset, *data->mData);
     }
     if (data->mCommands & SourceMediaStream::TRACK_CREATE) {
-      MediaSegment* segment = data->mData.forget();
+      MediaSegment* segment = data->mData->CreateEmptyClone();
       LOG(LogLevel::Debug,
           ("%p: SourceMediaStream %p creating track %d, start %" PRId64
            ", initial end %" PRId64,
            GraphImpl(), this, data->mID, int64_t(streamCurrentTime),
            int64_t(segment->GetDuration())));
 
-      segment->InsertNullDataAtStart(streamCurrentTime);
+      segment->AppendNullData(streamCurrentTime);
+      MoveToSegment(this, data->mData, segment, streamCurrentTime,
+                    streamDesiredUpToTime);
       data->mEndOfFlushedData += segment->GetDuration();
       mTracks.AddTrack(data->mID, streamCurrentTime, segment);
-      // The track has taken ownership of data->mData, so let's replace
-      // data->mData with an empty clone.
-      data->mData = segment->CreateEmptyClone();
       data->mCommands &= ~SourceMediaStream::TRACK_CREATE;
     } else {
-      MediaSegment* dest = mTracks.FindTrack(data->mID)->GetSegment();
+      StreamTracks::Track* track = mTracks.FindTrack(data->mID);
+      MediaSegment* dest = track->GetSegment();
       LOG(LogLevel::Verbose,
           ("%p: SourceMediaStream %p track %d, advancing end from %" PRId64
            " to %" PRId64,
            GraphImpl(), this, data->mID, int64_t(dest->GetDuration()),
            int64_t(dest->GetDuration() + data->mData->GetDuration())));
       data->mEndOfFlushedData += data->mData->GetDuration();
-      dest->AppendFrom(data->mData);
+      MoveToSegment(this, data->mData, dest, streamCurrentTime,
+                    streamDesiredUpToTime);
     }
     if (data->mCommands & SourceMediaStream::TRACK_END) {
       mTracks.FindTrack(data->mID)->SetEnded();
       mUpdateTracks.RemoveElementAt(i);
-    } else if (!data->mPullingEnabled &&
-               data->mData->GetType() == MediaSegment::VIDEO) {
-      // This video track is pushed. Since we use timestamps rather than
-      // durations for video we avoid making the video track block the stream
-      // by extending the duration when there's not enough video data, so a
-      // video track always has valid data.
-      VideoSegment* segment = static_cast<VideoSegment*>(
-          mTracks.FindTrack(data->mID)->GetSegment());
-      StreamTime missingTime = streamDesiredUpToTime - segment->GetDuration();
-      segment->ExtendLastFrameBy(missingTime);
     }
   }
 
   if (mTracks.GetEarliestTrackEnd() > 0) {
     mHasCurrentData = true;
   }
 
   if (finished) {