Bug 1229987: P2. Drain decoder when encountering gap. r=cpearce a=sylvestre
authorJean-Yves Avenard <jyavenard@mozilla.com>
Tue, 08 Dec 2015 14:30:25 -0500
changeset 298055 bab2c729fe29
parent 298054 c1a3159224ae
child 298056 9fa906dfe06d
push id8857
push userkwierso@gmail.com
push dateMon, 28 Dec 2015 16:46:13 +0000
treeherdermozilla-aurora@5628bb302871 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerscpearce, sylvestre
bugs1229987
milestone45.0a2
Bug 1229987: P2. Drain decoder when encountering gap. r=cpearce a=sylvestre This allows for all buffered frames to be playable.
dom/media/MediaFormatReader.cpp
dom/media/MediaFormatReader.h
--- a/dom/media/MediaFormatReader.cpp
+++ b/dom/media/MediaFormatReader.cpp
@@ -541,16 +541,19 @@ MediaFormatReader::OnDemuxFailed(TrackTy
   switch (aFailure) {
     case DemuxerFailureReason::END_OF_STREAM:
       NotifyEndOfStream(aTrack);
       break;
     case DemuxerFailureReason::DEMUXER_ERROR:
       NotifyError(aTrack);
       break;
     case DemuxerFailureReason::WAITING_FOR_DATA:
+      if (!decoder.mWaitingForData) {
+        decoder.mNeedDraining = true;
+      }
       NotifyWaitingForData(aTrack);
       break;
     case DemuxerFailureReason::CANCELED:
     case DemuxerFailureReason::SHUTDOWN:
       if (decoder.HasPromise()) {
         decoder.RejectPromise(CANCELED, __func__);
       }
       break;
@@ -686,16 +689,19 @@ MediaFormatReader::NotifyError(TrackType
 }
 
 void
 MediaFormatReader::NotifyWaitingForData(TrackType aTrack)
 {
   MOZ_ASSERT(OnTaskQueue());
   auto& decoder = GetDecoderData(aTrack);
   decoder.mWaitingForData = true;
+  if (decoder.mTimeThreshold) {
+    decoder.mTimeThreshold.ref().mWaiting = true;
+  }
   ScheduleUpdate(aTrack);
 }
 
 void
 MediaFormatReader::NotifyEndOfStream(TrackType aTrack)
 {
   MOZ_ASSERT(OnTaskQueue());
   auto& decoder = GetDecoderData(aTrack);
@@ -755,28 +761,42 @@ bool
 MediaFormatReader::UpdateReceivedNewData(TrackType aTrack)
 {
   MOZ_ASSERT(OnTaskQueue());
   auto& decoder = GetDecoderData(aTrack);
 
   if (!decoder.mReceivedNewData) {
     return false;
   }
-  decoder.mReceivedNewData = false;
-  decoder.mWaitingForData = false;
-  bool hasLastEnd;
-  media::TimeUnit lastEnd = decoder.mTimeRanges.GetEnd(&hasLastEnd);
+
   // Update our cached TimeRange.
   decoder.mTimeRanges = decoder.mTrackDemuxer->GetBuffered();
-  if (decoder.mTimeRanges.Length() &&
-      (!hasLastEnd || decoder.mTimeRanges.GetEnd() > lastEnd)) {
-    // New data was added after our previous end, we can clear the EOS flag.
-    decoder.mDemuxEOS = false;
+
+  if (decoder.mDrainComplete || decoder.mDraining) {
+    // We do not want to clear mWaitingForData or mDemuxEOS while
+    // a drain is in progress in order to properly complete the operation.
+    return false;
   }
 
+  bool hasLastEnd;
+  media::TimeUnit lastEnd = decoder.mTimeRanges.GetEnd(&hasLastEnd);
+  if (hasLastEnd) {
+    if (decoder.mLastTimeRangesEnd && decoder.mLastTimeRangesEnd.ref() > lastEnd) {
+      // New data was added after our previous end, we can clear the EOS flag.
+      decoder.mDemuxEOS = false;
+    }
+    decoder.mLastTimeRangesEnd = Some(lastEnd);
+  }
+
+  decoder.mReceivedNewData = false;
+  if (decoder.mTimeThreshold) {
+    decoder.mTimeThreshold.ref().mWaiting = false;
+  }
+  decoder.mWaitingForData = false;
+
   if (decoder.mError) {
     return false;
   }
   if (decoder.HasWaitingPromise()) {
     MOZ_ASSERT(!decoder.HasPromise());
     LOG("We have new data. Resolving WaitingPromise");
     decoder.mWaitingPromise.Resolve(decoder.mType, __func__);
     return true;
@@ -803,16 +823,18 @@ MediaFormatReader::RequestDemuxSamples(T
 
   if (!decoder.mQueuedSamples.IsEmpty()) {
     // No need to demux new samples.
     return;
   }
 
   if (decoder.mDemuxEOS) {
     // Nothing left to demux.
+    // We do not want to attempt to demux while in waiting for data mode
+    // as it would retrigger an unecessary drain.
     return;
   }
 
   LOGV("Requesting extra demux %s", TrackTypeToStr(aTrack));
   if (aTrack == TrackInfo::kVideoTrack) {
     DoDemuxVideo();
   } else {
     DoDemuxAudio();
@@ -878,57 +900,31 @@ MediaFormatReader::HandleDemuxedSamples(
         return;
       }
 
       LOG("%s stream id has changed from:%d to:%d, recreating decoder.",
           TrackTypeToStr(aTrack), decoder.mLastStreamSourceID,
           info->GetID());
       decoder.mInfo = info;
       decoder.mLastStreamSourceID = info->GetID();
+      decoder.mNextStreamSourceID.reset();
       // Flush will clear our array of queued samples. So make a copy now.
       nsTArray<RefPtr<MediaRawData>> samples{decoder.mQueuedSamples};
       Flush(aTrack);
       decoder.mDecoder->Shutdown();
       decoder.mDecoder = nullptr;
       if (sample->mKeyframe) {
         decoder.mQueuedSamples.AppendElements(Move(samples));
         NotifyDecodingRequested(aTrack);
       } else {
-        MOZ_ASSERT(decoder.mTimeThreshold.isNothing());
+        SeekTarget seekTarget =
+          decoder.mTimeThreshold.refOr(SeekTarget(TimeUnit::FromMicroseconds(sample->mTime), false));
         LOG("Stream change occurred on a non-keyframe. Seeking to:%lld",
-            sample->mTime);
-        decoder.mTimeThreshold = Some(TimeUnit::FromMicroseconds(sample->mTime));
-        RefPtr<MediaFormatReader> self = this;
-        decoder.ResetDemuxer();
-        decoder.mSeekRequest.Begin(decoder.mTrackDemuxer->Seek(decoder.mTimeThreshold.ref())
-                   ->Then(OwnerThread(), __func__,
-                          [self, aTrack] (media::TimeUnit aTime) {
-                            auto& decoder = self->GetDecoderData(aTrack);
-                            decoder.mSeekRequest.Complete();
-                            self->NotifyDecodingRequested(aTrack);
-                          },
-                          [self, aTrack] (DemuxerFailureReason aResult) {
-                            auto& decoder = self->GetDecoderData(aTrack);
-                            decoder.mSeekRequest.Complete();
-                            switch (aResult) {
-                              case DemuxerFailureReason::WAITING_FOR_DATA:
-                                self->NotifyWaitingForData(aTrack);
-                                break;
-                              case DemuxerFailureReason::END_OF_STREAM:
-                                self->NotifyEndOfStream(aTrack);
-                                break;
-                              case DemuxerFailureReason::CANCELED:
-                              case DemuxerFailureReason::SHUTDOWN:
-                                break;
-                              default:
-                                self->NotifyError(aTrack);
-                                break;
-                            }
-                            decoder.mTimeThreshold.reset();
-                          }));
+            seekTarget.mTime.ToMicroseconds());
+        InternalSeek(aTrack, seekTarget);
       }
       return;
     }
 
     LOGV("Input:%lld (dts:%lld kf:%d)",
          sample->mTime, sample->mTimecode, sample->mKeyframe);
     decoder.mOutputRequested = true;
     decoder.mNumSamplesInput++;
@@ -953,16 +949,52 @@ MediaFormatReader::HandleDemuxedSamples(
     samplesPending = true;
   }
 
   // We have serviced the decoder's request for more data.
   decoder.mInputExhausted = false;
 }
 
 void
+MediaFormatReader::InternalSeek(TrackType aTrack, const SeekTarget& aTarget)
+{
+  MOZ_ASSERT(OnTaskQueue());
+  auto& decoder = GetDecoderData(aTrack);
+  decoder.mTimeThreshold = Some(aTarget);
+  RefPtr<MediaFormatReader> self = this;
+  decoder.ResetDemuxer();
+  decoder.mSeekRequest.Begin(decoder.mTrackDemuxer->Seek(decoder.mTimeThreshold.ref().mTime)
+             ->Then(OwnerThread(), __func__,
+                    [self, aTrack] (media::TimeUnit aTime) {
+                      auto& decoder = self->GetDecoderData(aTrack);
+                      decoder.mSeekRequest.Complete();
+                      self->NotifyDecodingRequested(aTrack);
+                    },
+                    [self, aTrack] (DemuxerFailureReason aResult) {
+                      auto& decoder = self->GetDecoderData(aTrack);
+                      decoder.mSeekRequest.Complete();
+                      switch (aResult) {
+                        case DemuxerFailureReason::WAITING_FOR_DATA:
+                          self->NotifyWaitingForData(aTrack);
+                          break;
+                        case DemuxerFailureReason::END_OF_STREAM:
+                          self->NotifyEndOfStream(aTrack);
+                          break;
+                        case DemuxerFailureReason::CANCELED:
+                        case DemuxerFailureReason::SHUTDOWN:
+                          break;
+                        default:
+                          self->NotifyError(aTrack);
+                          break;
+                      }
+                      decoder.mTimeThreshold.reset();
+                    }));
+}
+
+void
 MediaFormatReader::DrainDecoder(TrackType aTrack)
 {
   MOZ_ASSERT(OnTaskQueue());
 
   auto& decoder = GetDecoderData(aTrack);
   if (!decoder.mNeedDraining || decoder.mDraining) {
     return;
   }
@@ -987,36 +1019,29 @@ MediaFormatReader::Update(TrackType aTra
   MOZ_ASSERT(OnTaskQueue());
 
   if (mShutdown) {
     return;
   }
 
   LOGV("Processing update for %s", TrackTypeToStr(aTrack));
 
-  bool needInput = false;
   bool needOutput = false;
   auto& decoder = GetDecoderData(aTrack);
   decoder.mUpdateScheduled = false;
 
   if (!mInitDone) {
     return;
   }
 
   if (UpdateReceivedNewData(aTrack)) {
     LOGV("Nothing more to do");
     return;
   }
 
-  if (!decoder.HasPromise() && decoder.mWaitingForData) {
-    // Nothing more we can do at present.
-    LOGV("Still waiting for data.");
-    return;
-  }
-
   // Record number of frames decoded and parsed. Automatically update the
   // stats counters using the AutoNotifyDecoded stack-based class.
   AbstractMediaDecoder::AutoNotifyDecoded a(mDecoder);
 
   if (aTrack == TrackInfo::kVideoTrack) {
     uint64_t delta =
       decoder.mNumSamplesOutputTotal - mLastReportedNumDecodedFrames;
     a.mDecoded = static_cast<uint32_t>(delta);
@@ -1031,67 +1056,96 @@ MediaFormatReader::Update(TrackType aTra
         nsCString error;
         mVideo.mIsHardwareAccelerated =
           mVideo.mDecoder && mVideo.mDecoder->IsHardwareAccelerated(error);
       }
       while (decoder.mOutput.Length()) {
         RefPtr<MediaData> output = decoder.mOutput[0];
         decoder.mOutput.RemoveElementAt(0);
         decoder.mSizeOfQueue -= 1;
-        if (decoder.mTimeThreshold.isNothing() ||
-            media::TimeUnit::FromMicroseconds(output->mTime) >= decoder.mTimeThreshold.ref()) {
-          ReturnOutput(output, aTrack);
+        decoder.mLastSampleTime =
+          Some(media::TimeUnit::FromMicroseconds(output->mTime));
+        auto target = decoder.mTimeThreshold;
+        auto time = media::TimeUnit::FromMicroseconds(output->mTime);
+        if (target && time >= target.ref().mTime) {
+          // We have reached our internal seek target.
           decoder.mTimeThreshold.reset();
-          break;
+        }
+        if (target && (time < target.ref().mTime || target.ref().mDropTarget)) {
+          LOGV("Internal Seeking: Dropping %s frame time:%f wanted:%f (kf:%d)",
+               TrackTypeToStr(aTrack),
+               media::TimeUnit::FromMicroseconds(output->mTime).ToSeconds(),
+               target.ref().mTime.ToSeconds(),
+               output->mKeyframe);
         } else {
-          LOGV("Internal Seeking: Dropping frame time:%f wanted:%f (kf:%d)",
-               media::TimeUnit::FromMicroseconds(output->mTime).ToSeconds(),
-               decoder.mTimeThreshold.ref().ToSeconds(),
-               output->mKeyframe);
+           ReturnOutput(output, aTrack);
+           break;
         }
       }
-    } else if (decoder.mDrainComplete) {
+    }
+  }
+  if (decoder.HasPromise() && decoder.mOutput.IsEmpty()) {
+    if (decoder.mError) {
+      decoder.RejectPromise(DECODE_ERROR, __func__);
+      return;
+    }
+    if (decoder.mDrainComplete) {
+      bool wasDraining = decoder.mDraining;
       decoder.mDrainComplete = false;
       decoder.mDraining = false;
-      if (decoder.mError) {
-        LOG("Decoding Error");
-        decoder.RejectPromise(DECODE_ERROR, __func__);
-        return;
-      } else if (decoder.mDemuxEOS) {
+      if (decoder.mDemuxEOS) {
         decoder.RejectPromise(END_OF_STREAM, __func__);
+      } else if (decoder.mWaitingForData) {
+        if (wasDraining && decoder.mLastSampleTime &&
+            !decoder.mNextStreamSourceID) {
+          // We have completed draining the decoder following WaitingForData.
+          // Set up the internal seek machinery to be able to resume from the
+          // last sample decoded.
+          LOG("Seeking to last sample time: %lld",
+              decoder.mLastSampleTime.ref().ToMicroseconds());
+          InternalSeek(aTrack, SeekTarget(decoder.mLastSampleTime.ref(), true));
+        }
+        decoder.RejectPromise(WAITING_FOR_DATA, __func__);
       }
-    } else if (decoder.mError) {
-      decoder.RejectPromise(DECODE_ERROR, __func__);
-      return;
-    } else if (decoder.mWaitingForData) {
-      LOG("Waiting For Data");
-      decoder.RejectPromise(WAITING_FOR_DATA, __func__);
-      return;
+      // Now that draining has completed, we check if we have received
+      // new data again as the result may now be different from the earlier
+      // run.
+      if (UpdateReceivedNewData(aTrack)) {
+        LOGV("Nothing more to do");
+        return;
+      }
     }
   }
 
   if (decoder.mNeedDraining) {
     DrainDecoder(aTrack);
     return;
   }
 
-  if (!NeedInput(decoder)) {
+  bool needInput = NeedInput(decoder);
+
+  LOGV("Update(%s) ni=%d no=%d ie=%d, in:%llu out:%llu qs=%u pending:%u waiting:%d ahead:%d sid:%u",
+       TrackTypeToStr(aTrack), needInput, needOutput, decoder.mInputExhausted,
+       decoder.mNumSamplesInput, decoder.mNumSamplesOutput,
+       uint32_t(size_t(decoder.mSizeOfQueue)), uint32_t(decoder.mOutput.Length()),
+       decoder.mWaitingForData, !decoder.HasPromise(), decoder.mLastStreamSourceID);
+
+  if (decoder.mWaitingForData &&
+      (!decoder.mTimeThreshold || decoder.mTimeThreshold.ref().mWaiting)) {
+    // Nothing more we can do at present.
+    LOGV("Still waiting for data.");
+    return;
+  }
+
+  if (!needInput) {
     LOGV("No need for additional input (pending:%u)",
          uint32_t(decoder.mOutput.Length()));
     return;
   }
 
-  needInput = true;
-
-  LOGV("Update(%s) ni=%d no=%d ie=%d, in:%llu out:%llu qs=%u pending:%u ahead:%d sid:%u",
-       TrackTypeToStr(aTrack), needInput, needOutput, decoder.mInputExhausted,
-       decoder.mNumSamplesInput, decoder.mNumSamplesOutput,
-       uint32_t(size_t(decoder.mSizeOfQueue)), uint32_t(decoder.mOutput.Length()),
-       !decoder.HasPromise(), decoder.mLastStreamSourceID);
-
   // Demux samples if we don't have some.
   RequestDemuxSamples(aTrack);
 
   HandleDemuxedSamples(aTrack, a);
 }
 
 void
 MediaFormatReader::ReturnOutput(MediaData* aData, TrackType aTrack)
--- a/dom/media/MediaFormatReader.h
+++ b/dom/media/MediaFormatReader.h
@@ -122,16 +122,32 @@ private:
   // Called when new samples need to be demuxed.
   void RequestDemuxSamples(TrackType aTrack);
   // Handle demuxed samples by the input behavior.
   void HandleDemuxedSamples(TrackType aTrack,
                             AbstractMediaDecoder::AutoNotifyDecoded& aA);
   // Decode any pending already demuxed samples.
   bool DecodeDemuxedSamples(TrackType aTrack,
                             MediaRawData* aSample);
+
+  struct SeekTarget {
+    SeekTarget(const media::TimeUnit& aTime, bool aDropTarget)
+      : mTime(aTime)
+      , mDropTarget(aDropTarget)
+      , mWaiting(false)
+    {}
+
+    media::TimeUnit mTime;
+    bool mDropTarget;
+    bool mWaiting;
+  };
+  // Perform an internal seek to aTime. If aDropTarget is true then
+  // the first sample past the target will be dropped.
+  void InternalSeek(TrackType aTrack, const SeekTarget& aTarget);
+
   // Drain the current decoder.
   void DrainDecoder(TrackType aTrack);
   void NotifyNewOutput(TrackType aTrack, MediaData* aSample);
   void NotifyInputExhausted(TrackType aTrack);
   void NotifyDrainComplete(TrackType aTrack);
   void NotifyError(TrackType aTrack);
   void NotifyWaitingForData(TrackType aTrack);
   void NotifyEndOfStream(TrackType aTrack);
@@ -256,18 +272,21 @@ private:
     bool mDecodingRequested;
     bool mOutputRequested;
     bool mInputExhausted;
     bool mError;
     bool mNeedDraining;
     bool mDraining;
     bool mDrainComplete;
     // If set, all decoded samples prior mTimeThreshold will be dropped.
-    // Used for internal seeking when a change of stream is detected.
-    Maybe<media::TimeUnit> mTimeThreshold;
+    // Used for internal seeking when a change of stream is detected or when
+    // encountering data discontinuity.
+    Maybe<SeekTarget> mTimeThreshold;
+    // Time of last sample returned.
+    Maybe<media::TimeUnit> mLastSampleTime;
 
     // Decoded samples returned my mDecoder awaiting being returned to
     // state machine upon request.
     nsTArray<RefPtr<MediaData>> mOutput;
     uint64_t mNumSamplesInput;
     uint64_t mNumSamplesOutput;
     uint64_t mNumSamplesOutputTotal;
 
@@ -295,32 +314,34 @@ private:
       mQueuedSamples.Clear();
       mDecodingRequested = false;
       mOutputRequested = false;
       mInputExhausted = false;
       mNeedDraining = false;
       mDraining = false;
       mDrainComplete = false;
       mTimeThreshold.reset();
+      mLastSampleTime.reset();
       mOutput.Clear();
       mNumSamplesInput = 0;
       mNumSamplesOutput = 0;
       mSizeOfQueue = 0;
       mNextStreamSourceID.reset();
     }
 
     // Used by the MDSM for logging purposes.
     Atomic<size_t> mSizeOfQueue;
     // Used by the MDSM to determine if video decoding is hardware accelerated.
     // This value is updated after a frame is successfully decoded.
     Atomic<bool> mIsHardwareAccelerated;
     // Sample format monitoring.
     uint32_t mLastStreamSourceID;
     Maybe<uint32_t> mNextStreamSourceID;
     media::TimeIntervals mTimeRanges;
+    Maybe<media::TimeUnit> mLastTimeRangesEnd;
     RefPtr<SharedTrackInfo> mInfo;
   };
 
   template<typename PromiseType>
   struct DecoderDataWithPromise : public DecoderData {
     DecoderDataWithPromise(MediaFormatReader* aOwner,
                            MediaData::Type aType,
                            uint32_t aDecodeAhead) :