Bug 1127554 - Make MP4Sample::Prepend fallible. v1 r=mattwoodrow
☠☠ backed out by ec501c1722c0 ☠ ☠
authorBobby Holley <bobbyholley@gmail.com>
Wed, 11 Feb 2015 15:12:01 -0800
changeset 249906 386a1b2123872aae11324592140bdf845c540c9a
parent 249905 bdeb39455a420d06d8de0eb35dfbea24e5e30a93
child 249907 abcc80b81681af8b3689875a2955c25078160081
push id4489
push userraliiev@mozilla.com
push dateMon, 23 Feb 2015 15:17:55 +0000
treeherdermozilla-beta@fd7c3dc24146 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmattwoodrow
bugs1127554
milestone37.0a2
Bug 1127554 - Make MP4Sample::Prepend fallible. v1 r=mattwoodrow
dom/media/fmp4/android/AndroidDecoderModule.cpp
dom/media/fmp4/ffmpeg/FFmpegH264Decoder.cpp
dom/media/fmp4/gonk/GonkVideoDecoderManager.cpp
dom/media/fmp4/wmf/WMFVideoMFTManager.cpp
media/libstagefright/binding/Adts.cpp
media/libstagefright/binding/AnnexB.cpp
media/libstagefright/binding/DecoderData.cpp
media/libstagefright/binding/include/mp4_demuxer/AnnexB.h
media/libstagefright/binding/include/mp4_demuxer/DecoderData.h
--- a/dom/media/fmp4/android/AndroidDecoderModule.cpp
+++ b/dom/media/fmp4/android/AndroidDecoderModule.cpp
@@ -61,17 +61,20 @@ public:
     return InitDecoder(mSurfaceTexture->JavaSurface());
   }
 
   void Cleanup() MOZ_OVERRIDE {
     mGLContext = nullptr;
   }
 
   virtual nsresult Input(mp4_demuxer::MP4Sample* aSample) MOZ_OVERRIDE {
-    mp4_demuxer::AnnexB::ConvertSampleToAnnexB(aSample);
+    if (!mp4_demuxer::AnnexB::ConvertSampleToAnnexB(aSample)) {
+      return NS_ERROR_OUT_OF_MEMORY;
+    }
+
     return MediaCodecDataDecoder::Input(aSample);
   }
 
   bool WantCopy() {
     // Allocating a texture is incredibly slow on PowerVR
     return mGLContext->Vendor() != GLVendor::Imagination;
   }
 
--- a/dom/media/fmp4/ffmpeg/FFmpegH264Decoder.cpp
+++ b/dom/media/fmp4/ffmpeg/FFmpegH264Decoder.cpp
@@ -48,17 +48,22 @@ FFmpegH264Decoder<LIBAV_VER>::Init()
 }
 
 FFmpegH264Decoder<LIBAV_VER>::DecodeResult
 FFmpegH264Decoder<LIBAV_VER>::DoDecodeFrame(mp4_demuxer::MP4Sample* aSample)
 {
   AVPacket packet;
   av_init_packet(&packet);
 
-  mp4_demuxer::AnnexB::ConvertSampleToAnnexB(aSample);
+  if (!mp4_demuxer::AnnexB::ConvertSampleToAnnexB(aSample)) {
+    NS_WARNING("FFmpeg h264 decoder failed to convert sample to Annex B.");
+    mCallback->Error();
+    return DecodeResult::DECODE_ERROR;
+  }
+
   if (!aSample->Pad(FF_INPUT_BUFFER_PADDING_SIZE)) {
     NS_WARNING("FFmpeg h264 decoder failed to allocate sample.");
     mCallback->Error();
     return DecodeResult::DECODE_ERROR;
   }
 
   packet.data = aSample->data;
   packet.size = aSample->size;
--- a/dom/media/fmp4/gonk/GonkVideoDecoderManager.cpp
+++ b/dom/media/fmp4/gonk/GonkVideoDecoderManager.cpp
@@ -440,17 +440,20 @@ GonkVideoDecoderManager::Input(mp4_demux
 {
   if (mDecoder == nullptr) {
     GVDM_LOG("Decoder is not inited");
     return NS_ERROR_UNEXPECTED;
   }
   status_t rv;
   if (aSample != nullptr) {
     // We must prepare samples in AVC Annex B.
-    mp4_demuxer::AnnexB::ConvertSampleToAnnexB(aSample);
+    if (!mp4_demuxer::AnnexB::ConvertSampleToAnnexB(aSample)) {
+      return NS_ERROR_FAILURE;
+    }
+
     // Forward sample data to the decoder.
 
     QueueFrameTimeIn(aSample->composition_timestamp, aSample->duration);
 
     const uint8_t* data = reinterpret_cast<const uint8_t*>(aSample->data);
     uint32_t length = aSample->size;
     rv = mDecoder->Input(data, length, aSample->composition_timestamp, 0);
   }
--- a/dom/media/fmp4/wmf/WMFVideoMFTManager.cpp
+++ b/dom/media/fmp4/wmf/WMFVideoMFTManager.cpp
@@ -231,17 +231,19 @@ WMFVideoMFTManager::Init()
   return decoder.forget();
 }
 
 HRESULT
 WMFVideoMFTManager::Input(mp4_demuxer::MP4Sample* aSample)
 {
   if (mStreamType != VP8 && mStreamType != VP9) {
     // We must prepare samples in AVC Annex B.
-    mp4_demuxer::AnnexB::ConvertSampleToAnnexB(aSample);
+    if (!mp4_demuxer::AnnexB::ConvertSampleToAnnexB(aSample)) {
+      return E_FAIL;
+    }
   }
   // Forward sample data to the decoder.
   const uint8_t* data = reinterpret_cast<const uint8_t*>(aSample->data);
   uint32_t length = aSample->size;
   return mDecoder->Input(data, length, aSample->composition_timestamp);
 }
 
 HRESULT
--- a/media/libstagefright/binding/Adts.cpp
+++ b/media/libstagefright/binding/Adts.cpp
@@ -51,17 +51,20 @@ Adts::ConvertSample(uint16_t aChannelCou
   header[1] = 0xf1;
   header[2] =
     ((aProfile - 1) << 6) + (aFrequencyIndex << 2) + (aChannelCount >> 2);
   header[3] = ((aChannelCount & 0x3) << 6) + (newSize >> 11);
   header[4] = (newSize & 0x7ff) >> 3;
   header[5] = ((newSize & 7) << 5) + 0x1f;
   header[6] = 0xfc;
 
-  aSample->Prepend(&header[0], ArrayLength(header));
+  if (!aSample->Prepend(&header[0], ArrayLength(header))) {
+    return false;
+  }
+
   if (aSample->crypto.valid) {
     if (aSample->crypto.plain_sizes.Length() == 0) {
       aSample->crypto.plain_sizes.AppendElement(kADTSHeaderSize);
       aSample->crypto.encrypted_sizes.AppendElement(aSample->size - kADTSHeaderSize);
     } else {
       aSample->crypto.plain_sizes[0] += kADTSHeaderSize;
     }
   }
--- a/media/libstagefright/binding/AnnexB.cpp
+++ b/media/libstagefright/binding/AnnexB.cpp
@@ -11,31 +11,31 @@
 
 using namespace mozilla;
 
 namespace mp4_demuxer
 {
 
 static const uint8_t kAnnexBDelimiter[] = { 0, 0, 0, 1 };
 
-void
+bool
 AnnexB::ConvertSampleToAnnexB(MP4Sample* aSample)
 {
   MOZ_ASSERT(aSample);
 
   if (!IsAVCC(aSample)) {
-    return;
+    return true;
   }
   MOZ_ASSERT(aSample->data);
 
   ConvertSampleTo4BytesAVCC(aSample);
 
   if (aSample->size < 4) {
-    // Nothing to do, it's corrupted anyway.
-    return;
+    // Nothing to do, it's corrupted anyway.
+    return true;
   }
 
   ByteReader reader(aSample->data, aSample->size);
 
   mozilla::Vector<uint8_t> tmp;
   ByteWriter writer(tmp);
 
   while (reader.Remaining() >= 4) {
@@ -50,18 +50,22 @@ AnnexB::ConvertSampleToAnnexB(MP4Sample*
   }
 
   aSample->Replace(tmp.begin(), tmp.length());
 
   // Prepend the Annex B NAL with SPS and PPS tables to keyframes.
   if (aSample->is_sync_point) {
     nsRefPtr<ByteBuffer> annexB =
       ConvertExtraDataToAnnexB(aSample->extra_data);
-    aSample->Prepend(annexB->Elements(), annexB->Length());
+    if (!aSample->Prepend(annexB->Elements(), annexB->Length())) {
+      return false;
+    }
   }
+
+  return true;
 }
 
 already_AddRefed<ByteBuffer>
 AnnexB::ConvertExtraDataToAnnexB(const ByteBuffer* aExtraData)
 {
   // AVCC 6 byte header looks like:
   //     +------+------+------+------+------+------+------+------+
   // [0] |   0  |   0  |   0  |   0  |   0  |   0  |   0  |   1  |
--- a/media/libstagefright/binding/DecoderData.cpp
+++ b/media/libstagefright/binding/DecoderData.cpp
@@ -277,38 +277,43 @@ MP4Sample::Pad(size_t aPaddingBytes)
       mMediaBuffer->release();
       mMediaBuffer = nullptr;
     }
   }
 
   return true;
 }
 
-void
+bool
 MP4Sample::Prepend(const uint8_t* aData, size_t aSize)
 {
   size_t newSize = size + aSize;
 
   // If the existing MediaBuffer has enough space then we just recycle it. If
   // not then we copy to a new buffer.
   uint8_t* newData = mMediaBuffer && newSize <= mMediaBuffer->size()
                        ? data
-                       : new uint8_t[newSize];
+                       : new (fallible) uint8_t[newSize];
+  if (!newData) {
+    return false;
+  }
 
   memmove(newData + aSize, data, size);
   memmove(newData, aData, aSize);
   size = newSize;
 
   if (newData != data) {
     extra_buffer = data = newData;
     if (mMediaBuffer) {
       mMediaBuffer->release();
       mMediaBuffer = nullptr;
     }
   }
+
+  return true;
 }
 
 void
 MP4Sample::Replace(const uint8_t* aData, size_t aSize)
 {
   // If the existing MediaBuffer has enough space then we just recycle it. If
   // not then we copy to a new buffer.
   uint8_t* newData = mMediaBuffer && aSize <= mMediaBuffer->size()
--- a/media/libstagefright/binding/include/mp4_demuxer/AnnexB.h
+++ b/media/libstagefright/binding/include/mp4_demuxer/AnnexB.h
@@ -14,17 +14,17 @@ namespace mp4_demuxer
 class ByteReader;
 class MP4Sample;
 
 class AnnexB
 {
 public:
   // All conversions assume size of NAL length field is 4 bytes.
   // Convert a sample from AVCC format to Annex B.
-  static void ConvertSampleToAnnexB(MP4Sample* aSample);
+  static bool ConvertSampleToAnnexB(MP4Sample* aSample);
   // Convert a sample from Annex B to AVCC.
   // an AVCC extradata must not be set.
   static void ConvertSampleToAVCC(MP4Sample* aSample);
   static void ConvertSampleTo4BytesAVCC(MP4Sample* aSample);
 
   // Parse an AVCC extradata and construct the Annex B sample header.
   static already_AddRefed<ByteBuffer> ConvertExtraDataToAnnexB(
     const ByteBuffer* aExtraData);
--- a/media/libstagefright/binding/include/mp4_demuxer/DecoderData.h
+++ b/media/libstagefright/binding/include/mp4_demuxer/DecoderData.h
@@ -169,17 +169,17 @@ public:
   bool is_sync_point;
 
   uint8_t* data;
   size_t size;
 
   CryptoSample crypto;
   nsRefPtr<ByteBuffer> extra_data;
 
-  void Prepend(const uint8_t* aData, size_t aSize);
+  bool Prepend(const uint8_t* aData, size_t aSize);
   void Replace(const uint8_t* aData, size_t aSize);
 
   nsAutoArrayPtr<uint8_t> extra_buffer;
 private:
   MP4Sample(const MP4Sample&); // Not implemented
 };
 }