Bug 1274626 part 3 - make the blank video decoder return samples in PTS order; r=jya
authorKaku Kuo <tkuo@mozilla.com>
Wed, 03 Aug 2016 15:22:49 +0800
changeset 307886 a2b41bc89af12fe5e7dd6cfddd00cb964b4c26ce
parent 307885 876f15c4bfffec79a2d66e46704daf86df7a8c40
child 307887 fd97d47edbb9c3671694dc2446a131e3901f4a41
push id80212
push usercbook@mozilla.com
push dateWed, 03 Aug 2016 13:50:13 +0000
treeherdermozilla-inbound@91c0b3e80825 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjya
bugs1274626
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1274626 part 3 - make the blank video decoder return samples in PTS order; r=jya MozReview-Commit-ID: EULu34WIcw0
dom/media/platforms/ReorderQueue.h
dom/media/platforms/agnostic/BlankDecoderModule.cpp
dom/media/platforms/apple/AppleVTDecoder.cpp
dom/media/platforms/apple/ReorderQueue.h
media/libstagefright/binding/H264.cpp
media/libstagefright/binding/include/mp4_demuxer/H264.h
rename from dom/media/platforms/apple/ReorderQueue.h
rename to dom/media/platforms/ReorderQueue.h
--- a/dom/media/platforms/agnostic/BlankDecoderModule.cpp
+++ b/dom/media/platforms/agnostic/BlankDecoderModule.cpp
@@ -6,34 +6,41 @@
 
 #include "ImageContainer.h"
 #include "MediaDecoderReader.h"
 #include "MediaInfo.h"
 #include "mozilla/CheckedInt.h"
 #include "mozilla/mozalloc.h" // for operator new, and new (fallible)
 #include "mozilla/RefPtr.h"
 #include "mozilla/TaskQueue.h"
+#include "mp4_demuxer/H264.h"
+#include "MP4Decoder.h"
 #include "nsAutoPtr.h"
 #include "nsRect.h"
 #include "PlatformDecoderModule.h"
+#include "ReorderQueue.h"
 #include "TimeUnits.h"
 #include "VideoUtils.h"
 
 namespace mozilla {
 
 // Decoder that uses a passed in object's Create function to create blank
 // MediaData objects.
 template<class BlankMediaDataCreator>
 class BlankMediaDataDecoder : public MediaDataDecoder {
 public:
 
   BlankMediaDataDecoder(BlankMediaDataCreator* aCreator,
                         const CreateDecoderParams& aParams)
     : mCreator(aCreator)
     , mCallback(aParams.mCallback)
+    , mMaxRefFrames(aParams.mConfig.GetType() == TrackInfo::kVideoTrack &&
+                    MP4Decoder::IsH264(aParams.mConfig.mMimeType)
+                    ? mp4_demuxer::H264::ComputeMaxRefFrames(aParams.VideoConfig().mExtraData)
+                    : 0)
     , mType(aParams.mConfig.GetType())
   {
   }
 
   RefPtr<InitPromise> Init() override {
     return InitPromise::CreateAndResolve(mType, __func__);
   }
 
@@ -42,41 +49,70 @@ public:
   }
 
   nsresult Input(MediaRawData* aSample) override
   {
     RefPtr<MediaData> data =
       mCreator->Create(media::TimeUnit::FromMicroseconds(aSample->mTime),
                        media::TimeUnit::FromMicroseconds(aSample->mDuration),
                        aSample->mOffset);
-    if (!data) {
-    mCallback->Error(MediaDataDecoderError::FATAL_ERROR);
-    } else {
-      mCallback->Output(data);
-    }
+
+    OutputFrame(data);
+
     return NS_OK;
   }
 
-  nsresult Flush() override {
+  nsresult Flush() override
+  {
+    mReorderQueue.Clear();
+
     return NS_OK;
   }
 
-  nsresult Drain() override {
+  nsresult Drain() override
+  {
+    while (!mReorderQueue.IsEmpty()) {
+      mCallback->Output(mReorderQueue.Pop().get());
+    }
+
     mCallback->DrainComplete();
     return NS_OK;
   }
 
   const char* GetDescriptionName() const override
   {
     return "blank media data decoder";
   }
 
 private:
+  void OutputFrame(MediaData* aData)
+  {
+    if (!aData) {
+      mCallback->Error(MediaDataDecoderError::FATAL_ERROR);
+      return;
+    }
+
+    // Frames come out in DTS order but we need to output them in PTS order.
+    mReorderQueue.Push(aData);
+
+    while (mReorderQueue.Length() > mMaxRefFrames) {
+      mCallback->Output(mReorderQueue.Pop().get());
+    }
+
+    if (mReorderQueue.Length() <= mMaxRefFrames) {
+      mCallback->InputExhausted();
+    }
+
+  }
+
+private:
   nsAutoPtr<BlankMediaDataCreator> mCreator;
   MediaDataDecoderCallback* mCallback;
+  const uint32_t mMaxRefFrames;
+  ReorderQueue mReorderQueue;
   TrackInfo::TrackType mType;
 };
 
 class BlankVideoDataCreator {
 public:
   BlankVideoDataCreator(uint32_t aFrameWidth,
                         uint32_t aFrameHeight,
                         layers::ImageContainer* aImageContainer)
@@ -228,17 +264,21 @@ public:
                    DecoderDoctorDiagnostics* aDiagnostics) const override
   {
     return true;
   }
 
   ConversionRequired
   DecoderNeedsConversion(const TrackInfo& aConfig) const override
   {
-    return kNeedNone;
+    if (aConfig.IsVideo() && MP4Decoder::IsH264(aConfig.mMimeType)) {
+      return kNeedAVCC;
+    } else {
+      return kNeedNone;
+    }
   }
 
 };
 
 already_AddRefed<PlatformDecoderModule> CreateBlankDecoderModule()
 {
   RefPtr<PlatformDecoderModule> pdm = new BlankDecoderModule();
   return pdm.forget();
--- a/dom/media/platforms/apple/AppleVTDecoder.cpp
+++ b/dom/media/platforms/apple/AppleVTDecoder.cpp
@@ -6,58 +6,42 @@
 
 #include <CoreFoundation/CFString.h>
 
 #include "AppleCMLinker.h"
 #include "AppleDecoderModule.h"
 #include "AppleUtils.h"
 #include "AppleVTDecoder.h"
 #include "AppleVTLinker.h"
-#include "mp4_demuxer/H264.h"
 #include "MediaData.h"
 #include "mozilla/ArrayUtils.h"
+#include "mp4_demuxer/H264.h"
 #include "nsAutoPtr.h"
 #include "nsThreadUtils.h"
 #include "mozilla/Logging.h"
 #include "VideoUtils.h"
 #include "gfxPlatform.h"
 
 #define LOG(...) MOZ_LOG(sPDMLog, mozilla::LogLevel::Debug, (__VA_ARGS__))
 
 namespace mozilla {
 
-static uint32_t ComputeMaxRefFrames(const MediaByteBuffer* aExtraData)
-{
-  uint32_t maxRefFrames = 4;
-  // Retrieve video dimensions from H264 SPS NAL.
-  mp4_demuxer::SPSData spsdata;
-  if (mp4_demuxer::H264::DecodeSPSFromExtraData(aExtraData, spsdata)) {
-    // max_num_ref_frames determines the size of the sliding window
-    // we need to queue that many frames in order to guarantee proper
-    // pts frames ordering. Use a minimum of 4 to ensure proper playback of
-    // non compliant videos.
-    maxRefFrames =
-      std::min(std::max(maxRefFrames, spsdata.max_num_ref_frames + 1), 16u);
-  }
-  return maxRefFrames;
-}
-
 AppleVTDecoder::AppleVTDecoder(const VideoInfo& aConfig,
                                TaskQueue* aTaskQueue,
                                MediaDataDecoderCallback* aCallback,
                                layers::ImageContainer* aImageContainer)
   : mExtraData(aConfig.mExtraData)
   , mCallback(aCallback)
   , mPictureWidth(aConfig.mImage.width)
   , mPictureHeight(aConfig.mImage.height)
   , mDisplayWidth(aConfig.mDisplay.width)
   , mDisplayHeight(aConfig.mDisplay.height)
   , mQueuedSamples(0)
   , mTaskQueue(aTaskQueue)
-  , mMaxRefFrames(ComputeMaxRefFrames(aConfig.mExtraData))
+  , mMaxRefFrames(mp4_demuxer::H264::ComputeMaxRefFrames(aConfig.mExtraData))
   , mImageContainer(aImageContainer)
   , mInputIncoming(0)
   , mIsShutDown(false)
 #ifdef MOZ_WIDGET_UIKIT
   , mUseSoftwareImages(true)
 #else
   , mUseSoftwareImages(false)
 #endif
--- a/media/libstagefright/binding/H264.cpp
+++ b/media/libstagefright/binding/H264.cpp
@@ -526,9 +526,26 @@ H264::EnsureSPSIsSane(SPSData& aSPS)
   }
   if (aSPS.max_num_ref_frames > 16) {
     aSPS.max_num_ref_frames = 16;
     valid = false;
   }
   return valid;
 }
 
+/* static */ uint32_t
+H264::ComputeMaxRefFrames(const mozilla::MediaByteBuffer* aExtraData)
+{
+  uint32_t maxRefFrames = 4;
+  // Retrieve video dimensions from H264 SPS NAL.
+  SPSData spsdata;
+  if (DecodeSPSFromExtraData(aExtraData, spsdata)) {
+    // max_num_ref_frames determines the size of the sliding window
+    // we need to queue that many frames in order to guarantee proper
+    // pts frames ordering. Use a minimum of 4 to ensure proper playback of
+    // non compliant videos.
+    maxRefFrames =
+      std::min(std::max(maxRefFrames, spsdata.max_num_ref_frames + 1), 16u);
+  }
+  return maxRefFrames;
+}
+
 } // namespace mp4_demuxer
--- a/media/libstagefright/binding/include/mp4_demuxer/H264.h
+++ b/media/libstagefright/binding/include/mp4_demuxer/H264.h
@@ -327,27 +327,34 @@ struct SPSData
 
   SPSData();
 };
 
 class H264
 {
 public:
   static bool DecodeSPSFromExtraData(const mozilla::MediaByteBuffer* aExtraData, SPSData& aDest);
+
   /* Extract RAW BYTE SEQUENCE PAYLOAD from NAL content.
      Returns nullptr if invalid content.
      This is compliant to ITU H.264 7.3.1 Syntax in tabular form NAL unit syntax
    */
   static already_AddRefed<mozilla::MediaByteBuffer> DecodeNALUnit(const mozilla::MediaByteBuffer* aNAL);
+
   /* Decode SPS NAL RBSP and fill SPSData structure */
   static bool DecodeSPS(const mozilla::MediaByteBuffer* aSPS, SPSData& aDest);
+
   // Ensure that SPS data makes sense, Return true if SPS data was, and false
   // otherwise. If false, then content will be adjusted accordingly.
   static bool EnsureSPSIsSane(SPSData& aSPS);
 
+  // If the given aExtraData is valid, return the aExtraData.max_num_ref_frames
+  // clamped to be in the range of [4, 16]; otherwise return 4.
+  static uint32_t ComputeMaxRefFrames(const mozilla::MediaByteBuffer* aExtraData);
+
 private:
   static void vui_parameters(BitReader& aBr, SPSData& aDest);
   // Read HRD parameters, all data is ignored.
   static void hrd_parameters(BitReader& aBr);
 };
 
 } // namespace mp4_demuxer