Bug 1185115 - Ensure we have enough memory to allocate required capacity. r=kentuckyfriedtakahe

CLOSED TREE

media/libstagefright/frameworks/av/media/libstagefright/SampleTable.cpp

@@ -580,7 +580,9 @@ SampleTable::setSampleAuxiliaryInformationOffsetParams(
     }
     data_offset += 4;
 
-    mCencOffsets.setCapacity(cencOffsetCount);
+    if (mCencOffsets.setCapacity(cencOffsetCount) < 0) {
+        return ERROR_MALFORMED;
+    }
     if (!version) {
         for (uint32_t i = 0; i < cencOffsetCount; i++) {
             uint32_t tmp;
@@ -1104,8 +1106,12 @@ SampleTable::getSampleCencInfo(
     }
 
     auto& info = mCencInfo[sample_index];
-    clear_sizes.setCapacity(info.mSubsampleCount);
-    cipher_sizes.setCapacity(info.mSubsampleCount);
+    if (clear_sizes.setCapacity(info.mSubsampleCount) < 0) {
+        return ERROR_MALFORMED;
+    }
+    if (cipher_sizes.setCapacity(info.mSubsampleCount) < 0) {
+        return ERROR_MALFORMED;
+    }
 
     for (uint32_t i = 0; i < info.mSubsampleCount; i++) {
         clear_sizes.push(info.mSubsamples[i].mClearBytes);

media/libstagefright/system/core/include/utils/Vector.h

@@ -18,6 +18,7 @@
 #define ANDROID_VECTOR_H
 
 #include <new>
+#include <assert.h>
 #include <stdint.h>
 #include <sys/types.h>
 
@@ -191,7 +192,7 @@ public:
      inline iterator end()   { return editArray() + size(); }
      inline const_iterator begin() const { return array(); }
      inline const_iterator end() const   { return array() + size(); }
-     inline void reserve(size_t n) { setCapacity(n); }
+     inline void reserve(size_t n) { assert(setCapacity(n) >= 0); }
      inline bool empty() const{ return isEmpty(); }
      inline void push_back(const TYPE& item)  { insertAt(item, size(), 1); }
      inline void push_front(const TYPE& item) { insertAt(item, 0, 1); }

media/libstagefright/system/core/libutils/VectorImpl.cpp

@@ -27,9 +27,8 @@
 #include <utils/SharedBuffer.h>
 #include <utils/VectorImpl.h>
 
-#if !defined(SSIZE_MAX)
-#define SSIZE_MAX ((ssize_t)(SIZE_MAX/2))
-#endif
+static const uint32_t kMAX_ALLOCATION =
+    ((SIZE_MAX > INT32_MAX ? INT32_MAX : SIZE_MAX) - 1);
 
 /*****************************************************************************/
 
@@ -93,6 +92,7 @@ void* VectorImpl::editArrayImpl()
         SharedBuffer* sb = SharedBuffer::bufferFromData(mStorage)->attemptEdit();
         if (sb == 0) {
             sb = SharedBuffer::alloc(capacity() * mItemSize);
+            assert(sb);
             if (sb) {
                 _do_copy(sb->data(), mStorage, mCount);
                 release_storage();
@@ -334,7 +334,7 @@ ssize_t VectorImpl::setCapacity(size_t new_capacity)
         // we can't reduce the capacity
         return capacity();
     }
-    if (new_capacity >= (SSIZE_MAX / mItemSize)) {
+    if (new_capacity >= (kMAX_ALLOCATION / mItemSize)) {
         return NO_MEMORY;
     }
     SharedBuffer* sb = SharedBuffer::alloc(new_capacity * mItemSize);
@@ -380,8 +380,11 @@ void* VectorImpl::_grow(size_t where, size_t amount)
             this, (int)where, (int)amount, (int)mCount); // caller already checked
 
     const size_t new_size = mCount + amount;
+    assert(amount < kMAX_ALLOCATION - mCount);
     if (capacity() < new_size) {
+        assert(new_size < (SIZE_MAX / 3 - 1));
         const size_t new_capacity = max(kMinVectorCapacity, ((new_size*3)+1)/2);
+        assert(new_capacity < (kMAX_ALLOCATION / mItemSize));
 //        ALOGV("grow vector %p, new_capacity=%d", this, (int)new_capacity);
         if ((mStorage) &&
             (mCount==where) &&
@@ -389,10 +392,13 @@ void* VectorImpl::_grow(size_t where, size_t amount)
             (mFlags & HAS_TRIVIAL_DTOR))
         {
             const SharedBuffer* cur_sb = SharedBuffer::bufferFromData(mStorage);
+            assert(cur_sb);
             SharedBuffer* sb = cur_sb->editResize(new_capacity * mItemSize);
+            assert(sb);
             mStorage = sb->data();
         } else {
             SharedBuffer* sb = SharedBuffer::alloc(new_capacity * mItemSize);
+            assert(sb);
             if (sb) {
                 void* array = sb->data();
                 if (where != 0) {
@@ -433,18 +439,23 @@ void VectorImpl::_shrink(size_t where, size_t amount)
             this, (int)where, (int)amount, (int)mCount); // caller already checked
 
     const size_t new_size = mCount - amount;
-    if (new_size*3 < capacity()) {
+    assert(new_size < (SIZE_MAX / 2));
+    if (new_size*2 < capacity()) {
         const size_t new_capacity = max(kMinVectorCapacity, new_size*2);
 //        ALOGV("shrink vector %p, new_capacity=%d", this, (int)new_capacity);
+        assert(new_capacity < (kMAX_ALLOCATION / mItemSize));
         if ((where == new_size) &&
             (mFlags & HAS_TRIVIAL_COPY) &&
             (mFlags & HAS_TRIVIAL_DTOR))
         {
             const SharedBuffer* cur_sb = SharedBuffer::bufferFromData(mStorage);
+            assert(cur_sb);
             SharedBuffer* sb = cur_sb->editResize(new_capacity * mItemSize);
+            assert(sb);
             mStorage = sb->data();
         } else {
             SharedBuffer* sb = SharedBuffer::alloc(new_capacity * mItemSize);
+            assert(sb);
             if (sb) {
                 void* array = sb->data();
                 if (where != 0) {
@@ -461,6 +472,7 @@ void VectorImpl::_shrink(size_t where, size_t amount)
         }
     } else {
         void* array = editArrayImpl();
+        assert(array);
         void* to = reinterpret_cast<uint8_t *>(array) + where*mItemSize;
         _do_destroy(to, amount);
         if (where != new_size) {