Bug 1043695 - Add a frame reorder queue. r=cpearce
authorRalph Giles <giles@mozilla.com>
Thu, 24 Jul 2014 17:08:00 -0700
changeset 196828 26b80b058cd245285439408c388cb990ef3a4136
parent 196827 c62d33eb57d3b870cdc167c0cac1b442a36554d9
child 196829 dc0fbe8b84d29a9009275959e5688d21ce8005a2
push id46977
push userrgiles@mozilla.com
push dateWed, 30 Jul 2014 16:59:13 +0000
treeherdermozilla-inbound@dc0fbe8b84d2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerscpearce
bugs1043695
milestone34.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1043695 - Add a frame reorder queue. r=cpearce This fixes the frame drops on test.mp4. For now it just assumes a reorder depth of 3, but we need to plumb the actual value through the demuxer. Based on now-removed code from Edwin's FFmpegH264Decoder.
content/media/fmp4/apple/AppleVTDecoder.cpp
content/media/fmp4/apple/AppleVTDecoder.h
content/media/fmp4/apple/ReorderQueue.h
--- a/content/media/fmp4/apple/AppleVTDecoder.cpp
+++ b/content/media/fmp4/apple/AppleVTDecoder.cpp
@@ -105,28 +105,29 @@ AppleVTDecoder::Input(mp4_demuxer::MP4Sa
           &AppleVTDecoder::SubmitFrame,
           nsAutoPtr<mp4_demuxer::MP4Sample>(aSample)));
   return NS_OK;
 }
 
 nsresult
 AppleVTDecoder::Flush()
 {
+  mReorderQueue.Clear();
   return Drain();
 }
 
 nsresult
 AppleVTDecoder::Drain()
 {
   OSStatus rv = VTDecompressionSessionWaitForAsynchronousFrames(mSession);
   if (rv != noErr) {
     LOG("Error %d draining frames", rv);
     return NS_ERROR_FAILURE;
   }
-  return NS_OK;
+  return DrainReorderedFrames();
 }
 
 //
 // Implementation details.
 //
 
 // Context object to hold a copy of sample metadata.
 class FrameRef {
@@ -184,16 +185,25 @@ PlatformCallback(void* decompressionOutp
   MOZ_ASSERT(CFGetTypeID(image) == CVPixelBufferGetTypeID(),
     "VideoToolbox returned an unexpected image type");
 
   // Forward the data back to an object method which can access
   // the correct MP4Reader callback.
   decoder->OutputFrame(image, frameRef);
 }
 
+nsresult
+AppleVTDecoder::DrainReorderedFrames()
+{
+  while (!mReorderQueue.IsEmpty()) {
+    mCallback->Output(mReorderQueue.Pop());
+  }
+  return NS_OK;
+}
+
 // Copy and return a decoded frame.
 nsresult
 AppleVTDecoder::OutputFrame(CVPixelBufferRef aImage,
                             nsAutoPtr<FrameRef> aFrameRef)
 {
   size_t width = CVPixelBufferGetWidth(aImage);
   size_t height = CVPixelBufferGetHeight(aImage);
   LOG("  got decoded frame data... %ux%u %s", width, height,
@@ -265,17 +275,24 @@ AppleVTDecoder::OutputFrame(CVPixelBuffe
                       aFrameRef->duration,
                       buffer,
                       aFrameRef->is_sync_point,
                       aFrameRef->timestamp,
                       visible);
   // Unlock the returned image data.
   CVPixelBufferUnlockBaseAddress(aImage, kCVPixelBufferLock_ReadOnly);
 
-  mCallback->Output(data.forget());
+  // Frames come out in DTS order but we need to output them
+  // in composition order.
+  mReorderQueue.Push(data.forget());
+  if (mReorderQueue.Length() > 2) {
+    VideoData* readyData = mReorderQueue.Pop();
+    mCallback->Output(readyData);
+  }
+
   return NS_OK;
 }
 
 // Helper to fill in a timestamp structure.
 static CMSampleTimingInfo
 TimingInfoFromSample(mp4_demuxer::MP4Sample* aSample)
 {
   CMSampleTimingInfo timestamp;
--- a/content/media/fmp4/apple/AppleVTDecoder.h
+++ b/content/media/fmp4/apple/AppleVTDecoder.h
@@ -6,16 +6,18 @@
 
 #ifndef mozilla_AppleVTDecoder_h
 #define mozilla_AppleVTDecoder_h
 
 #include "PlatformDecoderModule.h"
 #include "mozilla/RefPtr.h"
 #include "mozilla/ReentrantMonitor.h"
 #include "nsIThread.h"
+#include "ReorderQueue.h"
+
 #include "VideoToolbox/VideoToolbox.h"
 
 namespace mozilla {
 
 class MediaTaskQueue;
 class MediaDataDecoderCallback;
 namespace layers {
   class ImageContainer;
@@ -39,18 +41,20 @@ public:
                        nsAutoPtr<FrameRef> frameRef);
 private:
   const mp4_demuxer::VideoDecoderConfig& mConfig;
   RefPtr<MediaTaskQueue> mTaskQueue;
   MediaDataDecoderCallback* mCallback;
   layers::ImageContainer* mImageContainer;
   CMVideoFormatDescriptionRef mFormat;
   VTDecompressionSessionRef mSession;
+  ReorderQueue mReorderQueue;
 
   // Method to pass a frame to VideoToolbox for decoding.
   nsresult SubmitFrame(mp4_demuxer::MP4Sample* aSample);
   // Method to set up the decompression session.
   nsresult InitializeSession();
+  nsresult DrainReorderedFrames();
 };
 
 } // namespace mozilla
 
 #endif // mozilla_AppleVTDecoder_h
new file mode 100644
--- /dev/null
+++ b/content/media/fmp4/apple/ReorderQueue.h
@@ -0,0 +1,29 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Queue for ordering decoded video frames by presentation time.
+// Decoders often return frames out of order, which we need to
+// buffer so we can forward them in correct presentation order.
+
+#ifndef mozilla_ReorderQueue_h
+#define mozilla_ReorderQueue_h
+
+#include <MediaData.h>
+#include <nsTPriorityQueue.h>
+
+namespace mozilla {
+
+struct ReorderQueueComparator
+{
+  bool LessThan(VideoData* const& a, VideoData* const& b) const
+  {
+    return a->mTime < b->mTime;
+  }
+};
+
+typedef nsTPriorityQueue<VideoData*, ReorderQueueComparator> ReorderQueue;
+
+} // namespace mozilla
+
+#endif // mozilla_ReorderQueue_h