Bug 1185115 - Ensure we have enough memory to allocate required capacity. r=kentuckyfriedtakahe, a=sledru
authorJean-Yves Avenard <jyavenard@mozilla.com>
Tue, 04 Aug 2015 13:43:17 -0400
changeset 291366 2a94276e39798dc8ac197fe81fd61b14b79ffabe
parent 291365 9a114caf7a2932f25b5a854818bf7907d86fbbc4
child 291367 33b63ac4046e17c57d726dda7d8609fdf14744c6
child 328010 7c736ab19b007c19aaf4232f3be29e5790850dec
push id5246
push usermozilla@noorenberghe.ca
push dateWed, 09 Sep 2015 21:17:14 +0000
reviewerskentuckyfriedtakahe, sledru
bugs1185115
milestone40.0
Bug 1185115 - Ensure we have enough memory to allocate required capacity. r=kentuckyfriedtakahe, a=sledru CLOSED TREE
media/libstagefright/frameworks/av/media/libstagefright/SampleTable.cpp
media/libstagefright/system/core/include/utils/Vector.h
media/libstagefright/system/core/libutils/VectorImpl.cpp
--- a/media/libstagefright/frameworks/av/media/libstagefright/SampleTable.cpp
+++ b/media/libstagefright/frameworks/av/media/libstagefright/SampleTable.cpp
@@ -575,17 +575,19 @@ SampleTable::setSampleAuxiliaryInformati
 
     uint32_t cencOffsetCount;
     if (!mDataSource->getUInt32(data_offset, &cencOffsetCount)) {
         ALOGE("error reading cenc aux info offset count");
         return ERROR_IO;
     }
     data_offset += 4;
 
-    mCencOffsets.setCapacity(cencOffsetCount);
+    if (mCencOffsets.setCapacity(cencOffsetCount) < 0) {
+        return ERROR_MALFORMED;
+    }
     if (!version) {
         for (uint32_t i = 0; i < cencOffsetCount; i++) {
             uint32_t tmp;
             if (!mDataSource->getUInt32(data_offset, &tmp)) {
                 ALOGE("error reading cenc aux info offsets");
                 return ERROR_IO;
             }
             mCencOffsets.push(tmp);
@@ -1099,18 +1101,22 @@ SampleTable::getSampleCencInfo(
     CHECK(clear_sizes.isEmpty() && cipher_sizes.isEmpty());
 
     if (sample_index >= mCencInfoCount) {
         ALOGE("cenc info requested for out of range sample index");
         return ERROR_MALFORMED;
     }
 
     auto& info = mCencInfo[sample_index];
-    clear_sizes.setCapacity(info.mSubsampleCount);
-    cipher_sizes.setCapacity(info.mSubsampleCount);
+    if (clear_sizes.setCapacity(info.mSubsampleCount) < 0) {
+        return ERROR_MALFORMED;
+    }
+    if (cipher_sizes.setCapacity(info.mSubsampleCount) < 0) {
+        return ERROR_MALFORMED;
+    }
 
     for (uint32_t i = 0; i < info.mSubsampleCount; i++) {
         clear_sizes.push(info.mSubsamples[i].mClearBytes);
         cipher_sizes.push(info.mSubsamples[i].mCipherBytes);
     }
 
     memcpy(iv, info.mIV, IV_BYTES);
 
--- a/media/libstagefright/system/core/include/utils/Vector.h
+++ b/media/libstagefright/system/core/include/utils/Vector.h
@@ -13,16 +13,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 
 #ifndef ANDROID_VECTOR_H
 #define ANDROID_VECTOR_H
 
 #include <new>
+#include <assert.h>
 #include <stdint.h>
 #include <sys/types.h>
 
 #include <cutils/log.h>
 
 #include <utils/VectorImpl.h>
 #include <utils/TypeHelpers.h>
 
@@ -186,17 +187,17 @@ public:
       */
      typedef TYPE* iterator;
      typedef TYPE const* const_iterator;
 
      inline iterator begin() { return editArray(); }
      inline iterator end()   { return editArray() + size(); }
      inline const_iterator begin() const { return array(); }
      inline const_iterator end() const   { return array() + size(); }
-     inline void reserve(size_t n) { setCapacity(n); }
+     inline void reserve(size_t n) { assert(setCapacity(n) >= 0); }
      inline bool empty() const{ return isEmpty(); }
      inline void push_back(const TYPE& item)  { insertAt(item, size(), 1); }
      inline void push_front(const TYPE& item) { insertAt(item, 0, 1); }
      inline iterator erase(iterator pos) {
          ssize_t index = removeItemsAt(pos-array());
          return begin() + index;
      }
 
--- a/media/libstagefright/system/core/libutils/VectorImpl.cpp
+++ b/media/libstagefright/system/core/libutils/VectorImpl.cpp
@@ -22,19 +22,18 @@
 #include <stdio.h>
 
 #include <cutils/log.h>
 
 #include <utils/Errors.h>
 #include <utils/SharedBuffer.h>
 #include <utils/VectorImpl.h>
 
-#if !defined(SSIZE_MAX)
-#define SSIZE_MAX ((ssize_t)(SIZE_MAX/2))
-#endif
+static const uint32_t kMAX_ALLOCATION =
+    ((SIZE_MAX > INT32_MAX ? INT32_MAX : SIZE_MAX) - 1);
 
 /*****************************************************************************/
 
 
 namespace stagefright {
 
 // ----------------------------------------------------------------------------
 
@@ -88,16 +87,17 @@ VectorImpl& VectorImpl::operator = (cons
 }
 
 void* VectorImpl::editArrayImpl()
 {
     if (mStorage) {
         SharedBuffer* sb = SharedBuffer::bufferFromData(mStorage)->attemptEdit();
         if (sb == 0) {
             sb = SharedBuffer::alloc(capacity() * mItemSize);
+            assert(sb);
             if (sb) {
                 _do_copy(sb->data(), mStorage, mCount);
                 release_storage();
                 mStorage = sb->data();
             }
         }
     }
     return mStorage;
@@ -329,17 +329,17 @@ const void* VectorImpl::itemLocation(siz
 }
 
 ssize_t VectorImpl::setCapacity(size_t new_capacity)
 {
     if (new_capacity <= size()) {
         // we can't reduce the capacity
         return capacity();
     }
-    if (new_capacity >= (SSIZE_MAX / mItemSize)) {
+    if (new_capacity >= (kMAX_ALLOCATION / mItemSize)) {
         return NO_MEMORY;
     }
     SharedBuffer* sb = SharedBuffer::alloc(new_capacity * mItemSize);
     if (sb) {
         void* array = sb->data();
         _do_copy(array, mStorage, size());
         release_storage();
         mStorage = const_cast<void*>(array);
@@ -375,29 +375,35 @@ void* VectorImpl::_grow(size_t where, si
 //    ALOGV("_grow(this=%p, where=%d, amount=%d) count=%d, capacity=%d",
 //        this, (int)where, (int)amount, (int)mCount, (int)capacity());
 
     ALOG_ASSERT(where <= mCount,
             "[%p] _grow: where=%d, amount=%d, count=%d",
             this, (int)where, (int)amount, (int)mCount); // caller already checked
 
     const size_t new_size = mCount + amount;
+    assert(amount < kMAX_ALLOCATION - mCount);
     if (capacity() < new_size) {
+        assert(new_size < (SIZE_MAX / 3 - 1));
         const size_t new_capacity = max(kMinVectorCapacity, ((new_size*3)+1)/2);
+        assert(new_capacity < (kMAX_ALLOCATION / mItemSize));
 //        ALOGV("grow vector %p, new_capacity=%d", this, (int)new_capacity);
         if ((mStorage) &&
             (mCount==where) &&
             (mFlags & HAS_TRIVIAL_COPY) &&
             (mFlags & HAS_TRIVIAL_DTOR))
         {
             const SharedBuffer* cur_sb = SharedBuffer::bufferFromData(mStorage);
+            assert(cur_sb);
             SharedBuffer* sb = cur_sb->editResize(new_capacity * mItemSize);
+            assert(sb);
             mStorage = sb->data();
         } else {
             SharedBuffer* sb = SharedBuffer::alloc(new_capacity * mItemSize);
+            assert(sb);
             if (sb) {
                 void* array = sb->data();
                 if (where != 0) {
                     _do_copy(array, mStorage, where);
                 }
                 if (where != mCount) {
                     const void* from = reinterpret_cast<const uint8_t *>(mStorage) + where*mItemSize;
                     void* dest = reinterpret_cast<uint8_t *>(array) + (where+amount)*mItemSize;
@@ -428,44 +434,50 @@ void VectorImpl::_shrink(size_t where, s
 //    ALOGV("_shrink(this=%p, where=%d, amount=%d) count=%d, capacity=%d",
 //        this, (int)where, (int)amount, (int)mCount, (int)capacity());
 
     ALOG_ASSERT(where + amount <= mCount,
             "[%p] _shrink: where=%d, amount=%d, count=%d",
             this, (int)where, (int)amount, (int)mCount); // caller already checked
 
     const size_t new_size = mCount - amount;
-    if (new_size*3 < capacity()) {
+    assert(new_size < (SIZE_MAX / 2));
+    if (new_size*2 < capacity()) {
         const size_t new_capacity = max(kMinVectorCapacity, new_size*2);
 //        ALOGV("shrink vector %p, new_capacity=%d", this, (int)new_capacity);
+        assert(new_capacity < (kMAX_ALLOCATION / mItemSize));
         if ((where == new_size) &&
             (mFlags & HAS_TRIVIAL_COPY) &&
             (mFlags & HAS_TRIVIAL_DTOR))
         {
             const SharedBuffer* cur_sb = SharedBuffer::bufferFromData(mStorage);
+            assert(cur_sb);
             SharedBuffer* sb = cur_sb->editResize(new_capacity * mItemSize);
+            assert(sb);
             mStorage = sb->data();
         } else {
             SharedBuffer* sb = SharedBuffer::alloc(new_capacity * mItemSize);
+            assert(sb);
             if (sb) {
                 void* array = sb->data();
                 if (where != 0) {
                     _do_copy(array, mStorage, where);
                 }
                 if (where != new_size) {
                     const void* from = reinterpret_cast<const uint8_t *>(mStorage) + (where+amount)*mItemSize;
                     void* dest = reinterpret_cast<uint8_t *>(array) + where*mItemSize;
                     _do_copy(dest, from, new_size - where);
                 }
                 release_storage();
                 mStorage = const_cast<void*>(array);
             }
         }
     } else {
         void* array = editArrayImpl();
+        assert(array);
         void* to = reinterpret_cast<uint8_t *>(array) + where*mItemSize;
         _do_destroy(to, amount);
         if (where != new_size) {
             const void* from = reinterpret_cast<uint8_t *>(array) + (where+amount)*mItemSize;
             _do_move_backward(to, from, new_size - where);
         }
     }
     mCount = new_size;