Bug 1222851 - Keep the one dropped frame prior the first one to prime the decoder. r=bryce
authorJean-Yves Avenard <jyavenard@mozilla.com>
Fri, 22 Feb 2019 09:58:18 +0000
changeset 518688 754776b7161cd56f78777a531e2e5c344bf6a899
parent 518687 1b377d3c27c385bd84a9885618102d9937e8176d
child 518689 f67eca6e9528589d0516916fd66f3e097597a3f5
push id10862
push userffxbld-merge
push dateMon, 11 Mar 2019 13:01:11 +0000
treeherdermozilla-beta@a2e7f5c935da [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbryce
bugs1222851
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1222851 - Keep the one dropped frame prior the first one to prime the decoder. r=bryce Some audio decoders, such as AAC and Opus have a need for a pre-roll content. As such, in order to be able to fully get the content of the first frame we keep the frame just prior that would have normally been dropped. We set this frame to have a duration of 1us so that it will be dropped later by the decoding pipeline. The starting time of the first frame is adjusted so that we have continuous data, without gap in the buffered range. Differential Revision: https://phabricator.services.mozilla.com/D20322
dom/media/mediasource/TrackBuffersManager.cpp
--- a/dom/media/mediasource/TrackBuffersManager.cpp
+++ b/dom/media/mediasource/TrackBuffersManager.cpp
@@ -1625,32 +1625,51 @@ void TrackBuffersManager::ProcessFrames(
 
   // Highest presentation time seen in samples block.
   TimeUnit highestSampleTime;
 
   if (aSamples.Length()) {
     aTrackData.mLastParsedEndTime = TimeUnit();
   }
 
+  auto addToSamples = [&](MediaRawData* aSample,
+                          const TimeInterval& aInterval) {
+    aSample->mTime = aInterval.mStart;
+    aSample->mDuration = aInterval.Length();
+    aSample->mTrackInfo = trackBuffer.mLastInfo;
+    samplesRange += aInterval;
+    sizeNewSamples += aSample->ComputedSizeOfIncludingThis();
+    samples.AppendElement(aSample);
+  };
+
+  // Will be set to the last frame dropped due to being outside mAppendWindow.
+  // It will be added prior the first following frame which can be added to the
+  // track buffer.
+  // This sample will be set with a duration of only 1us which will cause it to
+  // be dropped once returned by the decoder.
+  // This sample is required to "prime" the decoder so that the following frame
+  // can be fully decoded.
+  RefPtr<MediaRawData> previouslyDroppedSample;
   for (auto& sample : aSamples) {
     const TimeUnit sampleEndTime = sample->GetEndTime();
     if (sampleEndTime > aTrackData.mLastParsedEndTime) {
       aTrackData.mLastParsedEndTime = sampleEndTime;
     }
 
     // We perform step 10 right away as we can't do anything should a keyframe
     // be needed until we have one.
 
     // 10. If the need random access point flag on track buffer equals true,
     // then run the following steps:
     if (trackBuffer.mNeedRandomAccessPoint) {
       // 1. If the coded frame is not a random access point, then drop the coded
       // frame and jump to the top of the loop to start processing the next
       // coded frame.
       if (!sample->mKeyframe) {
+        previouslyDroppedSample = nullptr;
         continue;
       }
       // 2. Set the need random access point flag on track buffer to false.
       trackBuffer.mNeedRandomAccessPoint = false;
     }
 
     // We perform step 1,2 and 4 at once:
     // 1. If generate timestamps flag equals true:
@@ -1729,16 +1748,17 @@ void TrackBuffersManager::ProcessFrames(
       // 3. If mode equals "sequence" and group start timestamp is set, then run
       // the following steps:
       TimeUnit presentationTimestamp =
           mSourceBufferAttributes->mGenerateTimestamps ? TimeUnit()
                                                        : sampleTime;
       CheckSequenceDiscontinuity(presentationTimestamp);
 
       if (!sample->mKeyframe) {
+        previouslyDroppedSample = nullptr;
         continue;
       }
       if (appendMode == SourceBufferAppendMode::Sequence) {
         // mSourceBufferAttributes->GetTimestampOffset() was modified during
         // CheckSequenceDiscontinuity. We need to update our variables.
         timestampOffset = mSourceBufferAttributes->GetTimestampOffset();
         sampleInterval =
             mSourceBufferAttributes->mGenerateTimestamps
@@ -1788,18 +1808,20 @@ void TrackBuffersManager::ProcessFrames(
         sample->mOriginalPresentationWindow = Some(sampleInterval);
         MSE_DEBUGV("will truncate frame from [%" PRId64 ",%" PRId64
                    "] to [%" PRId64 ",%" PRId64 "]",
                    sampleInterval.mStart.ToMicroseconds(),
                    sampleInterval.mEnd.ToMicroseconds(),
                    intersection.mStart.ToMicroseconds(),
                    intersection.mEnd.ToMicroseconds());
         sampleInterval = intersection;
-        sample->mDuration = intersection.Length();
       } else {
+        sample->mOriginalPresentationWindow = Some(sampleInterval);
+        sample->mTimecode = decodeTimestamp;
+        previouslyDroppedSample = sample;
         MSE_DEBUGV("frame [%" PRId64 ",%" PRId64
                    "] outside appendWindow [%" PRId64 ",%" PRId64 "] dropping",
                    sampleInterval.mStart.ToMicroseconds(),
                    sampleInterval.mEnd.ToMicroseconds(),
                    mAppendWindow.mStart.ToMicroseconds(),
                    mAppendWindow.mEnd.ToMicroseconds());
         if (samples.Length()) {
           // We are creating a discontinuity in the samples.
@@ -1811,23 +1833,33 @@ void TrackBuffersManager::ProcessFrames(
           sizeNewSamples = 0;
           UpdateHighestTimestamp(trackBuffer, highestSampleTime);
         }
         trackBuffer.mNeedRandomAccessPoint = true;
         needDiscontinuityCheck = true;
         continue;
       }
     }
+    if (previouslyDroppedSample) {
+      MSE_DEBUGV("Adding silent frame");
+      // This "silent" sample will be added so that it starts exactly before the
+      // first usable one. The duration of the actual sample will be adjusted so
+      // that the total duration staty the same.
+      // Setting a dummy presentation window of 1us will cause this sample to be
+      // dropped after decoding by the AudioTrimmer (if audio).
+      TimeInterval previouslyDroppedSampleInterval =
+          TimeInterval(sampleInterval.mStart,
+                       sampleInterval.mStart + TimeUnit::FromMicroseconds(1));
+      addToSamples(previouslyDroppedSample, previouslyDroppedSampleInterval);
+      previouslyDroppedSample = nullptr;
+      sampleInterval.mStart += previouslyDroppedSampleInterval.Length();
+    }
 
-    samplesRange += sampleInterval;
-    sizeNewSamples += sample->ComputedSizeOfIncludingThis();
-    sample->mTime = sampleInterval.mStart;
     sample->mTimecode = decodeTimestamp;
-    sample->mTrackInfo = trackBuffer.mLastInfo;
-    samples.AppendElement(sample);
+    addToSamples(sample, sampleInterval);
 
     // Steps 11,12,13,14, 15 and 16 will be done in one block in InsertFrames.
 
     trackBuffer.mLongestFrameDuration =
         trackBuffer.mLastFrameDuration.isSome()
             ? sample->mKeyframe
                   ? sampleDuration
                   : std::max(sampleDuration, trackBuffer.mLongestFrameDuration)