Bug 1177278 - Large OOMs in CacheFileMetadata::WriteMetadata. r=honzab, a=lmandel
authorMichal Novotny <michal.novotny@gmail.com>
Wed, 22 Jul 2015 15:23:16 +0200
changeset 276797 5b3fbfa2cb9734f14c2e54d08f29cb0311d565b7
parent 276796 a047db3341e402ce263d87f8b59680a858e46351
child 276798 790d04f41b269f2095ae733c2ffa5c910268cbe7
push idunknown
push userunknown
push dateunknown
reviewershonzab, lmandel
bugs1177278
milestone41.0a2
Bug 1177278 - Large OOMs in CacheFileMetadata::WriteMetadata. r=honzab, a=lmandel
netwerk/cache2/CacheFileMetadata.cpp
netwerk/cache2/CacheFileMetadata.h
netwerk/cache2/CacheFileOutputStream.cpp
netwerk/cache2/CacheObserver.cpp
netwerk/cache2/CacheObserver.h
--- a/netwerk/cache2/CacheFileMetadata.cpp
+++ b/netwerk/cache2/CacheFileMetadata.cpp
@@ -27,16 +27,19 @@ namespace net {
 
 // Most of the cache entries fit into one chunk due to current chunk size. Make
 // sure to tweak this value if kChunkSize is going to change.
 #define kInitialHashArraySize 1
 
 // Initial elements buffer size.
 #define kInitialBufSize 64
 
+// Max size of elements in bytes.
+#define kMaxElementsSize 64*1024
+
 #define kCacheEntryVersion 1
 
 #define NOW_SECONDS() (uint32_t(PR_Now() / PR_USEC_PER_SEC))
 
 NS_IMPL_ISUPPORTS(CacheFileMetadata, CacheFileIOListener)
 
 CacheFileMetadata::CacheFileMetadata(CacheFileHandle *aHandle, const nsACString &aKey)
   : CacheMemoryConsumer(NORMAL)
@@ -231,34 +234,46 @@ CacheFileMetadata::ReadMetadata(CacheFil
     InitEmptyMetadata();
     aListener->OnMetadataRead(NS_OK);
     return NS_OK;
   }
 
   return NS_OK;
 }
 
+uint32_t
+CacheFileMetadata::CalcMetadataSize(uint32_t aElementsSize, uint32_t aHashCount)
+{
+  return sizeof(uint32_t) +                         // hash of the metadata
+         aHashCount * sizeof(CacheHash::Hash16_t) + // array of chunk hashes
+         sizeof(CacheFileMetadataHeader) +          // metadata header
+         mKey.Length() + 1 +                        // key with trailing null
+         aElementsSize +                            // elements
+         sizeof(uint32_t);                          // offset
+}
+
 nsresult
 CacheFileMetadata::WriteMetadata(uint32_t aOffset,
                                  CacheFileMetadataListener *aListener)
 {
   LOG(("CacheFileMetadata::WriteMetadata() [this=%p, offset=%d, listener=%p]",
        this, aOffset, aListener));
 
   MOZ_ASSERT(!mListener);
   MOZ_ASSERT(!mWriteBuf);
 
   nsresult rv;
 
   mIsDirty = false;
 
-  mWriteBuf = static_cast<char *>(moz_xmalloc(sizeof(uint32_t) +
-                mHashCount * sizeof(CacheHash::Hash16_t) +
-                sizeof(CacheFileMetadataHeader) + mKey.Length() + 1 +
-                mElementsSize + sizeof(uint32_t)));
+  mWriteBuf = static_cast<char *>(malloc(CalcMetadataSize(mElementsSize,
+                                                          mHashCount)));
+  if (!mWriteBuf) {
+    return NS_ERROR_OUT_OF_MEMORY;
+  }
 
   char *p = mWriteBuf + sizeof(uint32_t);
   memcpy(p, mHashArray, mHashCount * sizeof(CacheHash::Hash16_t));
   p += mHashCount * sizeof(CacheHash::Hash16_t);
   mMetaHdr.WriteToBuf(p);
   p += sizeof(CacheFileMetadataHeader);
   memcpy(p, mKey.get(), mKey.Length());
   p += mKey.Length();
@@ -401,16 +416,18 @@ CacheFileMetadata::GetElement(const char
 nsresult
 CacheFileMetadata::SetElement(const char *aKey, const char *aValue)
 {
   LOG(("CacheFileMetadata::SetElement() [this=%p, key=%s, value=%p]",
        this, aKey, aValue));
 
   MarkDirty();
 
+  nsresult rv;
+
   const uint32_t keySize = strlen(aKey) + 1;
   char *pos = const_cast<char *>(GetElement(aKey));
 
   if (!aValue) {
     // No value means remove the key/value pair completely, if existing
     if (pos) {
       uint32_t oldValueSize = strlen(pos) + 1;
       uint32_t offset = pos - mBuf;
@@ -426,25 +443,31 @@ CacheFileMetadata::SetElement(const char
   uint32_t newSize = mElementsSize + valueSize;
   if (pos) {
     const uint32_t oldValueSize = strlen(pos) + 1;
     const uint32_t offset = pos - mBuf;
     const uint32_t remainder = mElementsSize - (offset + oldValueSize);
 
     // Update the value in place
     newSize -= oldValueSize;
-    EnsureBuffer(newSize);
+    rv = EnsureBuffer(newSize);
+    if (NS_FAILED(rv)) {
+      return rv;
+    }
 
     // Move the remainder to the right place
     pos = mBuf + offset;
     memmove(pos + valueSize, pos + oldValueSize, remainder);
   } else {
     // allocate new meta data element
     newSize += keySize;
-    EnsureBuffer(newSize);
+    rv = EnsureBuffer(newSize);
+    if (NS_FAILED(rv)) {
+      return rv;
+    }
 
     // Add after last element
     pos = mBuf + mElementsSize;
     memcpy(pos, aKey, keySize);
     pos += keySize;
   }
 
   // Update value
@@ -660,26 +683,41 @@ CacheFileMetadata::OnDataRead(CacheFileH
   uint32_t realOffset = NetworkEndian::readUint32(mBuf + mBufSize -
                                                   sizeof(uint32_t));
 
   int64_t size = mHandle->FileSize();
   MOZ_ASSERT(size != -1);
 
   if (realOffset >= size) {
     LOG(("CacheFileMetadata::OnDataRead() - Invalid realOffset, creating "
-         "empty metadata. [this=%p, realOffset=%d, size=%lld]", this,
+         "empty metadata. [this=%p, realOffset=%u, size=%lld]", this,
          realOffset, size));
 
     InitEmptyMetadata();
 
     mListener.swap(listener);
     listener->OnMetadataRead(NS_OK);
     return NS_OK;
   }
 
+  uint32_t maxHashCount = size / kChunkSize;
+  uint32_t maxMetadataSize = CalcMetadataSize(kMaxElementsSize, maxHashCount);
+  if (size - realOffset > maxMetadataSize) {
+    LOG(("CacheFileMetadata::OnDataRead() - Invalid realOffset, metadata would "
+         "be too big, creating empty metadata. [this=%p, realOffset=%u, "
+         "maxMetadataSize=%u, size=%lld]", this, realOffset, maxMetadataSize,
+         size));
+
+    InitEmptyMetadata();
+
+    mListener.swap(listener);
+    listener->OnMetadataRead(NS_OK);
+    return NS_OK;
+  }
+
   uint32_t usedOffset = size - mBufSize;
 
   if (realOffset < usedOffset) {
     uint32_t missing = usedOffset - realOffset;
     // we need to read more data
     char *newBuf = static_cast<char *>(realloc(mBuf, mBufSize + missing));
     if (!newBuf) {
       LOG(("CacheFileMetadata::OnDataRead() - Error allocating %d more bytes "
@@ -927,19 +965,23 @@ CacheFileMetadata::CheckElements(const c
       LOG(("CacheFileMetadata::CheckElements() - Elements are malformed. "
            "[this=%p]", this));
       return NS_ERROR_FILE_CORRUPTED;
     }
   }
   return NS_OK;
 }
 
-void
+nsresult
 CacheFileMetadata::EnsureBuffer(uint32_t aSize)
 {
+  if (aSize > kMaxElementsSize) {
+    return NS_ERROR_FAILURE;
+  }
+
   if (mBufSize < aSize) {
     if (mAllocExactSize) {
       // If this is not the only allocation, use power of two for following
       // allocations.
       mAllocExactSize = false;
     } else {
       // find smallest power of 2 greater than or equal to aSize
       --aSize;
@@ -950,21 +992,27 @@ CacheFileMetadata::EnsureBuffer(uint32_t
       aSize |= aSize >> 16;
       ++aSize;
     }
 
     if (aSize < kInitialBufSize) {
       aSize = kInitialBufSize;
     }
 
+    char *newBuf = static_cast<char *>(realloc(mBuf, aSize));
+    if (!newBuf) {
+      return NS_ERROR_OUT_OF_MEMORY;
+    }
     mBufSize = aSize;
-    mBuf = static_cast<char *>(moz_xrealloc(mBuf, mBufSize));
+    mBuf = newBuf;
 
     DoMemoryReport(MemoryUsage());
   }
+
+  return NS_OK;
 }
 
 nsresult
 CacheFileMetadata::ParseKey(const nsACString &aKey)
 {
   nsCOMPtr<nsILoadContextInfo> info = CacheFileUtils::ParseKey(aKey);
   NS_ENSURE_TRUE(info, NS_ERROR_FAILURE);
 
--- a/netwerk/cache2/CacheFileMetadata.h
+++ b/netwerk/cache2/CacheFileMetadata.h
@@ -116,16 +116,17 @@ public:
                     const nsACString &aKey);
   CacheFileMetadata();
 
   void SetHandle(CacheFileHandle *aHandle);
 
   nsresult GetKey(nsACString &_retval);
 
   nsresult ReadMetadata(CacheFileMetadataListener *aListener);
+  uint32_t CalcMetadataSize(uint32_t aElementsSize, uint32_t aHashCount);
   nsresult WriteMetadata(uint32_t aOffset,
                          CacheFileMetadataListener *aListener);
   nsresult SyncReadMetadata(nsIFile *aFile);
 
   bool     IsAnonymous() { return mAnonymous; }
   bool     IsInBrowser() { return mInBrowser; }
   uint32_t AppId()       { return mAppId; }
 
@@ -166,17 +167,17 @@ public:
   size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
 
 private:
   virtual ~CacheFileMetadata();
 
   void     InitEmptyMetadata();
   nsresult ParseMetadata(uint32_t aMetaOffset, uint32_t aBufOffset, bool aHaveKey);
   nsresult CheckElements(const char *aBuf, uint32_t aSize);
-  void     EnsureBuffer(uint32_t aSize);
+  nsresult EnsureBuffer(uint32_t aSize);
   nsresult ParseKey(const nsACString &aKey);
 
   nsRefPtr<CacheFileHandle>           mHandle;
   nsCString                           mKey;
   CacheHash::Hash16_t                *mHashArray;
   uint32_t                            mHashArraySize;
   uint32_t                            mHashCount;
   int64_t                             mOffset;
--- a/netwerk/cache2/CacheFileOutputStream.cpp
+++ b/netwerk/cache2/CacheFileOutputStream.cpp
@@ -98,16 +98,28 @@ CacheFileOutputStream::Write(const char 
     LOG(("CacheFileOutputStream::Write() - Entry is too big, failing and "
          "dooming the entry. [this=%p]", this));
 
     mFile->DoomLocked(nullptr);
     CloseWithStatusLocked(NS_ERROR_FILE_TOO_BIG);
     return NS_ERROR_FILE_TOO_BIG;
   }
 
+  // We use 64-bit offset when accessing the file, unfortunatelly we use 32-bit
+  // metadata offset, so we cannot handle data bigger than 4GB.
+  if (mPos + aCount > PR_UINT32_MAX) {
+    LOG(("CacheFileOutputStream::Write() - Entry's size exceeds 4GB while it "
+         "isn't too big according to CacheObserver::EntryIsTooBig(). Failing "
+         "and dooming the entry. [this=%p]", this));
+
+    mFile->DoomLocked(nullptr);
+    CloseWithStatusLocked(NS_ERROR_FILE_TOO_BIG);
+    return NS_ERROR_FILE_TOO_BIG;
+  }
+
   *_retval = aCount;
 
   while (aCount) {
     EnsureCorrectChunk(false);
     if (NS_FAILED(mStatus)) {
       return mStatus;
     }
 
--- a/netwerk/cache2/CacheObserver.cpp
+++ b/netwerk/cache2/CacheObserver.cpp
@@ -60,21 +60,21 @@ static uint32_t const kDefaultDiskFreeSp
 uint32_t CacheObserver::sDiskFreeSpaceHardLimit = kDefaultDiskFreeSpaceHardLimit;
 
 static bool const kDefaultSmartCacheSizeEnabled = false;
 bool CacheObserver::sSmartCacheSizeEnabled = kDefaultSmartCacheSizeEnabled;
 
 static uint32_t const kDefaultPreloadChunkCount = 4;
 uint32_t CacheObserver::sPreloadChunkCount = kDefaultPreloadChunkCount;
 
-static uint32_t const kDefaultMaxMemoryEntrySize = 4 * 1024; // 4 MB
-uint32_t CacheObserver::sMaxMemoryEntrySize = kDefaultMaxMemoryEntrySize;
+static int32_t const kDefaultMaxMemoryEntrySize = 4 * 1024; // 4 MB
+int32_t CacheObserver::sMaxMemoryEntrySize = kDefaultMaxMemoryEntrySize;
 
-static uint32_t const kDefaultMaxDiskEntrySize = 50 * 1024; // 50 MB
-uint32_t CacheObserver::sMaxDiskEntrySize = kDefaultMaxDiskEntrySize;
+static int32_t const kDefaultMaxDiskEntrySize = 50 * 1024; // 50 MB
+int32_t CacheObserver::sMaxDiskEntrySize = kDefaultMaxDiskEntrySize;
 
 static uint32_t const kDefaultMaxDiskChunksMemoryUsage = 10 * 1024; // 10MB
 uint32_t CacheObserver::sMaxDiskChunksMemoryUsage = kDefaultMaxDiskChunksMemoryUsage;
 
 static uint32_t const kDefaultMaxDiskPriorityChunksMemoryUsage = 10 * 1024; // 10MB
 uint32_t CacheObserver::sMaxDiskPriorityChunksMemoryUsage = kDefaultMaxDiskPriorityChunksMemoryUsage;
 
 static uint32_t const kDefaultCompressionLevel = 1;
@@ -165,19 +165,19 @@ CacheObserver::AttachToPreferences()
   mozilla::Preferences::AddUintVarCache(
     &sDiskFreeSpaceSoftLimit, "browser.cache.disk.free_space_soft_limit", kDefaultDiskFreeSpaceSoftLimit);
   mozilla::Preferences::AddUintVarCache(
     &sDiskFreeSpaceHardLimit, "browser.cache.disk.free_space_hard_limit", kDefaultDiskFreeSpaceHardLimit);
 
   mozilla::Preferences::AddUintVarCache(
     &sPreloadChunkCount, "browser.cache.disk.preload_chunk_count", kDefaultPreloadChunkCount);
 
-  mozilla::Preferences::AddUintVarCache(
+  mozilla::Preferences::AddIntVarCache(
     &sMaxDiskEntrySize, "browser.cache.disk.max_entry_size", kDefaultMaxDiskEntrySize);
-  mozilla::Preferences::AddUintVarCache(
+  mozilla::Preferences::AddIntVarCache(
     &sMaxMemoryEntrySize, "browser.cache.memory.max_entry_size", kDefaultMaxMemoryEntrySize);
 
   mozilla::Preferences::AddUintVarCache(
     &sMaxDiskChunksMemoryUsage, "browser.cache.disk.max_chunks_memory_usage", kDefaultMaxDiskChunksMemoryUsage);
   mozilla::Preferences::AddUintVarCache(
     &sMaxDiskPriorityChunksMemoryUsage, "browser.cache.disk.max_priority_chunks_memory_usage", kDefaultMaxDiskPriorityChunksMemoryUsage);
 
   // http://mxr.mozilla.org/mozilla-central/source/netwerk/cache/nsCacheEntryDescriptor.cpp#367
@@ -467,19 +467,22 @@ CacheStorageEvictHelper::ClearStorage(bo
 }
 
 } // anon
 
 // static
 bool const CacheObserver::EntryIsTooBig(int64_t aSize, bool aUsingDisk)
 {
   // If custom limit is set, check it.
-  int64_t preferredLimit = aUsingDisk
-    ? static_cast<int64_t>(sMaxDiskEntrySize) << 10
-    : static_cast<int64_t>(sMaxMemoryEntrySize) << 10;
+  int64_t preferredLimit = aUsingDisk ? sMaxDiskEntrySize : sMaxMemoryEntrySize;
+
+  // do not convert to bytes when the limit is -1, which means no limit
+  if (preferredLimit > 0) {
+    preferredLimit <<= 10;
+  }
 
   if (preferredLimit != -1 && aSize > preferredLimit)
     return true;
 
   // Otherwise (or when in the custom limit), check limit based on the global
   // limit.  It's 1/8 (>> 3) of the respective capacity.
   int64_t derivedLimit = aUsingDisk
     ? (static_cast<int64_t>(DiskCacheCapacity() >> 3))
--- a/netwerk/cache2/CacheObserver.h
+++ b/netwerk/cache2/CacheObserver.h
@@ -85,18 +85,18 @@ private:
   static uint32_t sMetadataMemoryLimit;
   static int32_t sMemoryCacheCapacity;
   static int32_t sAutoMemoryCacheCapacity;
   static uint32_t sDiskCacheCapacity;
   static uint32_t sDiskFreeSpaceSoftLimit;
   static uint32_t sDiskFreeSpaceHardLimit;
   static bool sSmartCacheSizeEnabled;
   static uint32_t sPreloadChunkCount;
-  static uint32_t sMaxMemoryEntrySize;
-  static uint32_t sMaxDiskEntrySize;
+  static int32_t sMaxMemoryEntrySize;
+  static int32_t sMaxDiskEntrySize;
   static uint32_t sMaxDiskChunksMemoryUsage;
   static uint32_t sMaxDiskPriorityChunksMemoryUsage;
   static uint32_t sCompressionLevel;
   static float sHalfLifeHours;
   static int32_t sHalfLifeExperiment;
   static bool sSanitizeOnShutdown;
   static bool sClearCacheOnShutdown;
   static bool sCacheFSReported;