Bug 1013395 - HTTP cache v2: have a limit for write backlog, r=honzab
authorMichal Novotny <michal.novotny@gmail.com>
Thu, 10 Jul 2014 07:59:29 +0200
changeset 193246 8e969cdb22512e1a8466f5734b0f601dfafdcd9e
parent 193245 7ac8d502c5948902b7ccacc6d10d013b20764a92
child 193247 0f0e44cd5fbd5bd266eef6fa06ea1241823c164b
push id27112
push usercbook@mozilla.com
push dateThu, 10 Jul 2014 12:47:23 +0000
treeherdermozilla-central@6e9f72bdd32e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewershonzab
bugs1013395
milestone33.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1013395 - HTTP cache v2: have a limit for write backlog, r=honzab
modules/libpref/src/init/all.js
netwerk/cache2/CacheFile.cpp
netwerk/cache2/CacheFile.h
netwerk/cache2/CacheFileChunk.cpp
netwerk/cache2/CacheFileChunk.h
netwerk/cache2/CacheFileInputStream.cpp
netwerk/cache2/CacheFileInputStream.h
netwerk/cache2/CacheFileOutputStream.cpp
netwerk/cache2/CacheIndex.cpp
netwerk/cache2/CacheObserver.cpp
netwerk/cache2/CacheObserver.h
netwerk/protocol/http/nsHttpChannel.cpp
netwerk/test/unit/test_cache2-25-chunk-memory-limit.js
netwerk/test/unit/xpcshell.ini
toolkit/components/telemetry/Histograms.json
--- a/modules/libpref/src/init/all.js
+++ b/modules/libpref/src/init/all.js
@@ -53,16 +53,27 @@ pref("browser.cache.disk.capacity",     
 // (Note: entries bigger than 1/8 of disk-cache are never cached)
 pref("browser.cache.disk.max_entry_size",    51200);  // 50 MB
 pref("browser.cache.memory.enable",         true);
 // -1 = determine dynamically, 0 = none, n = memory capacity in kilobytes
 //pref("browser.cache.memory.capacity",     -1);
 // Max-size (in KB) for entries in memory cache. Set to -1 for no limit.
 // (Note: entries bigger than than 90% of the mem-cache are never cached)
 pref("browser.cache.memory.max_entry_size",  5120);
+// Memory limit (in kB) for new cache data not yet written to disk. Writes to
+// the cache are buffered and written to disk on background with low priority.
+// With a slow persistent storage these buffers may grow when data is coming
+// fast from the network. When the amount of unwritten data is exceeded, new
+// writes will simply fail. We have two buckets, one for important data
+// (priority) like html, css, fonts and js, and one for other data like images,
+// video, etc.
+// Note: 0 means no limit.
+pref("browser.cache.disk.max_chunks_memory_usage", 10240);
+pref("browser.cache.disk.max_priority_chunks_memory_usage", 10240);
+
 pref("browser.cache.disk_cache_ssl",        true);
 // 0 = once-per-session, 1 = each-time, 2 = never, 3 = when-appropriate/automatically
 pref("browser.cache.check_doc_frequency",   3);
 // Limit of recent metadata we keep in memory for faster access, in Kb
 pref("browser.cache.disk.metadata_memory_limit", 250); // 0.25 MB
 // The number of chunks we preload ahead of read.  One chunk has currently 256kB.
 pref("browser.cache.disk.preload_chunk_count", 4); // 1 MB of read ahead
 // The half life used to re-compute cache entries frecency in hours.
--- a/netwerk/cache2/CacheFile.cpp
+++ b/netwerk/cache2/CacheFile.cpp
@@ -8,16 +8,17 @@
 #include "CacheFileChunk.h"
 #include "CacheFileInputStream.h"
 #include "CacheFileOutputStream.h"
 #include "nsThreadUtils.h"
 #include "mozilla/DebugOnly.h"
 #include <algorithm>
 #include "nsComponentManagerUtils.h"
 #include "nsProxyRelease.h"
+#include "mozilla/Telemetry.h"
 
 // When CACHE_CHUNKS is defined we always cache unused chunks in mCacheChunks.
 // When it is not defined, we always release the chunks ASAP, i.e. we cache
 // unused chunks only when:
 //  - CacheFile is memory-only
 //  - CacheFile is still waiting for the handle
 //  - the chunk is preloaded
 
@@ -173,35 +174,27 @@ NS_INTERFACE_MAP_BEGIN(CacheFile)
 NS_INTERFACE_MAP_END_THREADSAFE
 
 CacheFile::CacheFile()
   : mLock("CacheFile.mLock")
   , mOpeningFile(false)
   , mReady(false)
   , mMemoryOnly(false)
   , mOpenAsMemoryOnly(false)
+  , mPriority(false)
   , mDataAccessed(false)
   , mDataIsDirty(false)
   , mWritingMetadata(false)
   , mPreloadWithoutInputStreams(true)
+  , mPreloadChunkCount(0)
   , mStatus(NS_OK)
   , mDataSize(-1)
   , mOutput(nullptr)
 {
   LOG(("CacheFile::CacheFile() [this=%p]", this));
-
-  // Some consumers (at least nsHTTPCompressConv) assume that Read() can read
-  // such amount of data that was announced by Available().
-  // CacheFileInputStream::Available() uses also preloaded chunks to compute
-  // number of available bytes in the input stream, so we have to make sure the
-  // preloadChunkCount won't change during CacheFile's lifetime since otherwise
-  // we could potentially release some cached chunks that was used to calculate
-  // available bytes but would not be available later during call to
-  // CacheFileInputStream::Read().
-  mPreloadChunkCount = CacheObserver::PreloadChunkCount();
 }
 
 CacheFile::~CacheFile()
 {
   LOG(("CacheFile::~CacheFile() [this=%p]", this));
 
   MutexAutoLock lock(mLock);
   if (!mMemoryOnly && mReady) {
@@ -219,19 +212,31 @@ CacheFile::Init(const nsACString &aKey,
 {
   MOZ_ASSERT(!mListener);
   MOZ_ASSERT(!mHandle);
 
   nsresult rv;
 
   mKey = aKey;
   mOpenAsMemoryOnly = mMemoryOnly = aMemoryOnly;
+  mPriority = aPriority;
+
+  // Some consumers (at least nsHTTPCompressConv) assume that Read() can read
+  // such amount of data that was announced by Available().
+  // CacheFileInputStream::Available() uses also preloaded chunks to compute
+  // number of available bytes in the input stream, so we have to make sure the
+  // preloadChunkCount won't change during CacheFile's lifetime since otherwise
+  // we could potentially release some cached chunks that was used to calculate
+  // available bytes but would not be available later during call to
+  // CacheFileInputStream::Read().
+  mPreloadChunkCount = CacheObserver::PreloadChunkCount();
 
   LOG(("CacheFile::Init() [this=%p, key=%s, createNew=%d, memoryOnly=%d, "
-       "listener=%p]", this, mKey.get(), aCreateNew, aMemoryOnly, aCallback));
+       "priority=%d, listener=%p]", this, mKey.get(), aCreateNew, aMemoryOnly,
+       aPriority, aCallback));
 
   if (mMemoryOnly) {
     MOZ_ASSERT(!aCallback);
 
     mMetadata = new CacheFileMetadata(mOpenAsMemoryOnly, mKey);
     mReady = true;
     mDataSize = mMetadata->Offset();
     return NS_OK;
@@ -245,18 +250,19 @@ CacheFile::Init(const nsACString &aKey,
       // make sure we can use this entry immediately
       mMetadata = new CacheFileMetadata(mOpenAsMemoryOnly, mKey);
       mReady = true;
       mDataSize = mMetadata->Offset();
     } else {
       flags = CacheFileIOManager::CREATE;
     }
 
-    if (aPriority)
+    if (mPriority) {
       flags |= CacheFileIOManager::PRIORITY;
+    }
 
     mOpeningFile = true;
     mListener = aCallback;
     rv = CacheFileIOManager::OpenFile(mKey, flags, true, this);
     if (NS_FAILED(rv)) {
       mListener = nullptr;
       mOpeningFile = false;
 
@@ -302,17 +308,16 @@ CacheFile::OnChunkRead(nsresult aResult,
 
   uint32_t index = aChunk->Index();
 
   LOG(("CacheFile::OnChunkRead() [this=%p, rv=0x%08x, chunk=%p, idx=%u]",
        this, aResult, aChunk, index));
 
   if (NS_FAILED(aResult)) {
     SetError(aResult);
-    CacheFileIOManager::DoomFile(mHandle, nullptr);
   }
 
   if (HaveChunkListeners(index)) {
     rv = NotifyChunkListeners(index, aResult, aChunk);
     NS_ENSURE_SUCCESS(rv, rv);
   }
 
   return NS_OK;
@@ -329,17 +334,16 @@ CacheFile::OnChunkWritten(nsresult aResu
        this, aResult, aChunk, aChunk->Index()));
 
   MOZ_ASSERT(!mMemoryOnly);
   MOZ_ASSERT(!mOpeningFile);
   MOZ_ASSERT(mHandle);
 
   if (NS_FAILED(aResult)) {
     SetError(aResult);
-    CacheFileIOManager::DoomFile(mHandle, nullptr);
   }
 
   if (NS_SUCCEEDED(aResult) && !aChunk->IsDirty()) {
     // update hash value in metadata
     mMetadata->SetHash(aChunk->Index(), aChunk->Hash());
   }
 
   // notify listeners if there is any
@@ -489,16 +493,19 @@ CacheFile::OnFileOpened(CacheFileHandle 
         isNew = false;
         retval = aResult;
       }
 
       mListener.swap(listener);
     }
     else {
       mHandle = aHandle;
+      if (NS_FAILED(mStatus)) {
+        CacheFileIOManager::DoomFile(mHandle, nullptr);
+      }
 
       if (mMetadata) {
         InitIndexEntry();
 
         // The entry was initialized as createNew, don't try to read metadata.
         mMetadata->SetHandle(mHandle);
 
         // Write all cached chunks, otherwise they may stay unwritten.
@@ -1077,17 +1084,17 @@ CacheFile::GetChunkLocked(uint32_t aInde
       // a null handle.
       LOG(("CacheFile::GetChunkLocked() - Unexpected state! Offset < mDataSize "
            "for memory-only entry. [this=%p, off=%lld, mDataSize=%lld]",
            this, off, mDataSize));
 
       return NS_ERROR_UNEXPECTED;
     }
 
-    chunk = new CacheFileChunk(this, aIndex);
+    chunk = new CacheFileChunk(this, aIndex, aCaller == WRITER);
     mChunks.Put(aIndex, chunk);
     chunk->mActiveChunk = true;
 
     LOG(("CacheFile::GetChunkLocked() - Reading newly created chunk %p from "
          "the disk [this=%p]", chunk.get(), this));
 
     // Read the chunk from the disk
     rv = chunk->Read(mHandle, std::min(static_cast<uint32_t>(mDataSize - off),
@@ -1108,24 +1115,24 @@ CacheFile::GetChunkLocked(uint32_t aInde
     if (preload) {
       PreloadChunks(aIndex + 1);
     }
 
     return NS_OK;
   } else if (off == mDataSize) {
     if (aCaller == WRITER) {
       // this listener is going to write to the chunk
-      chunk = new CacheFileChunk(this, aIndex);
+      chunk = new CacheFileChunk(this, aIndex, true);
       mChunks.Put(aIndex, chunk);
       chunk->mActiveChunk = true;
 
       LOG(("CacheFile::GetChunkLocked() - Created new empty chunk %p [this=%p]",
            chunk.get(), this));
 
-      chunk->InitNew(this);
+      chunk->InitNew();
       mMetadata->SetHash(aIndex, chunk->Hash());
 
       if (HaveChunkListeners(aIndex)) {
         rv = NotifyChunkListeners(aIndex, NS_OK, chunk);
         NS_ENSURE_SUCCESS(rv, rv);
       }
 
       chunk.swap(*_retval);
@@ -1335,16 +1342,20 @@ CacheFile::DeactivateChunk(CacheFileChun
 
       // We also shouldn't have any queued listener for this chunk
       ChunkListeners *listeners;
       mChunkListeners.Get(chunk->Index(), &listeners);
       MOZ_ASSERT(!listeners);
     }
 #endif
 
+    if (NS_FAILED(chunk->GetStatus())) {
+      SetError(chunk->GetStatus());
+    }
+
     if (NS_FAILED(mStatus)) {
       // Don't write any chunk to disk since this entry will be doomed
       LOG(("CacheFile::DeactivateChunk() - Releasing chunk because of status "
            "[this=%p, chunk=%p, mStatus=0x%08x]", this, chunk.get(), mStatus));
 
       RemoveChunkInternal(chunk, false);
       return mStatus;
     }
@@ -1359,17 +1370,16 @@ CacheFile::DeactivateChunk(CacheFileChun
       if (NS_FAILED(rv)) {
         LOG(("CacheFile::DeactivateChunk() - CacheFileChunk::Write() failed "
              "synchronously. Removing it. [this=%p, chunk=%p, rv=0x%08x]",
              this, chunk.get(), rv));
 
         RemoveChunkInternal(chunk, false);
 
         SetError(rv);
-        CacheFileIOManager::DoomFile(mHandle, nullptr);
         return rv;
       }
 
       // Chunk will be removed in OnChunkWritten if it is still unused
 
       // chunk needs to be released under the lock to be able to rely on
       // CacheFileChunk::mRefCnt in CacheFile::OnChunkWritten()
       chunk = nullptr;
@@ -1455,45 +1465,77 @@ CacheFile::BytesFromChunk(uint32_t aInde
   // theoretic bytes in advance
   int64_t advance = int64_t(i - aIndex) * kChunkSize;
   // real bytes till the end of the file
   int64_t tail = mDataSize - (aIndex * kChunkSize);
 
   return std::min(advance, tail);
 }
 
+static uint32_t
+StatusToTelemetryEnum(nsresult aStatus)
+{
+  if (NS_SUCCEEDED(aStatus)) {
+    return 0;
+  }
+
+  switch (aStatus) {
+    case NS_BASE_STREAM_CLOSED:
+      return 0; // Log this as a success
+    case NS_ERROR_OUT_OF_MEMORY:
+      return 2;
+    case NS_ERROR_FILE_DISK_FULL:
+      return 3;
+    case NS_ERROR_FILE_CORRUPTED:
+      return 4;
+    case NS_ERROR_FILE_NOT_FOUND:
+      return 5;
+    case NS_BINDING_ABORTED:
+      return 6;
+    default:
+      return 1; // other error
+  }
+
+  NS_NOTREACHED("We should never get here");
+}
+
 nsresult
-CacheFile::RemoveInput(CacheFileInputStream *aInput)
+CacheFile::RemoveInput(CacheFileInputStream *aInput, nsresult aStatus)
 {
   CacheFileAutoLock lock(this);
 
-  LOG(("CacheFile::RemoveInput() [this=%p, input=%p]", this, aInput));
+  LOG(("CacheFile::RemoveInput() [this=%p, input=%p, status=0x%08x]", this,
+       aInput, aStatus));
 
   DebugOnly<bool> found;
   found = mInputs.RemoveElement(aInput);
   MOZ_ASSERT(found);
 
   ReleaseOutsideLock(static_cast<nsIInputStream*>(aInput));
 
   if (!mMemoryOnly)
     WriteMetadataIfNeededLocked();
 
   // If the input didn't read all data, there might be left some preloaded
   // chunks that won't be used anymore.
   mCachedChunks.Enumerate(&CacheFile::CleanUpCachedChunks, this);
 
+  Telemetry::Accumulate(Telemetry::NETWORK_CACHE_V2_INPUT_STREAM_STATUS,
+                        StatusToTelemetryEnum(aStatus));
+
   return NS_OK;
 }
 
 nsresult
-CacheFile::RemoveOutput(CacheFileOutputStream *aOutput)
+CacheFile::RemoveOutput(CacheFileOutputStream *aOutput, nsresult aStatus)
 {
   AssertOwnsLock();
 
-  LOG(("CacheFile::RemoveOutput() [this=%p, output=%p]", this, aOutput));
+  LOG(("CacheFile::RemoveOutput() [this=%p, output=%p, status=0x%08x]", this,
+       aOutput, aStatus));
 
   if (mOutput != aOutput) {
     LOG(("CacheFile::RemoveOutput() - This output was already removed, ignoring"
          " call [this=%p]", this));
     return NS_OK;
   }
 
   mOutput = nullptr;
@@ -1502,16 +1544,19 @@ CacheFile::RemoveOutput(CacheFileOutputS
   NotifyListenersAboutOutputRemoval();
 
   if (!mMemoryOnly)
     WriteMetadataIfNeededLocked();
 
   // Notify close listener as the last action
   aOutput->NotifyCloseListener();
 
+  Telemetry::Accumulate(Telemetry::NETWORK_CACHE_V2_OUTPUT_STREAM_STATUS,
+                        StatusToTelemetryEnum(aStatus));
+
   return NS_OK;
 }
 
 nsresult
 CacheFile::NotifyChunkListener(CacheFileChunkListener *aCallback,
                                nsIEventTarget *aTarget,
                                nsresult aResult,
                                uint32_t aChunkIdx,
@@ -1844,18 +1889,23 @@ CacheFile::PadChunkWithZeroes(uint32_t a
   ReleaseOutsideLock(chunk.forget().take());
 
   return NS_OK;
 }
 
 void
 CacheFile::SetError(nsresult aStatus)
 {
+  AssertOwnsLock();
+
   if (NS_SUCCEEDED(mStatus)) {
     mStatus = aStatus;
+    if (mHandle) {
+      CacheFileIOManager::DoomFile(mHandle, nullptr);
+    }
   }
 }
 
 nsresult
 CacheFile::InitIndexEntry()
 {
   MOZ_ASSERT(mHandle);
 
--- a/netwerk/cache2/CacheFile.h
+++ b/netwerk/cache2/CacheFile.h
@@ -140,18 +140,18 @@ private:
   bool     ShouldCacheChunk(uint32_t aIndex);
   bool     MustKeepCachedChunk(uint32_t aIndex);
 
   nsresult DeactivateChunk(CacheFileChunk *aChunk);
   void     RemoveChunkInternal(CacheFileChunk *aChunk, bool aCacheChunk);
 
   int64_t  BytesFromChunk(uint32_t aIndex);
 
-  nsresult RemoveInput(CacheFileInputStream *aInput);
-  nsresult RemoveOutput(CacheFileOutputStream *aOutput);
+  nsresult RemoveInput(CacheFileInputStream *aInput, nsresult aStatus);
+  nsresult RemoveOutput(CacheFileOutputStream *aOutput, nsresult aStatus);
   nsresult NotifyChunkListener(CacheFileChunkListener *aCallback,
                                nsIEventTarget *aTarget,
                                nsresult aResult,
                                uint32_t aChunkIdx,
                                CacheFileChunk *aChunk);
   nsresult QueueChunkListener(uint32_t aIndex,
                               CacheFileChunkListener *aCallback);
   nsresult NotifyChunkListeners(uint32_t aIndex, nsresult aResult,
@@ -187,16 +187,17 @@ private:
 
   nsresult InitIndexEntry();
 
   mozilla::Mutex mLock;
   bool           mOpeningFile;
   bool           mReady;
   bool           mMemoryOnly;
   bool           mOpenAsMemoryOnly;
+  bool           mPriority;
   bool           mDataAccessed;
   bool           mDataIsDirty;
   bool           mWritingMetadata;
   bool           mPreloadWithoutInputStreams;
   uint32_t       mPreloadChunkCount;
   nsresult       mStatus;
   int64_t        mDataSize;
   nsCString      mKey;
--- a/netwerk/cache2/CacheFileChunk.cpp
+++ b/netwerk/cache2/CacheFileChunk.cpp
@@ -96,102 +96,119 @@ CacheFileChunk::Release()
   return count;
 }
 
 NS_INTERFACE_MAP_BEGIN(CacheFileChunk)
   NS_INTERFACE_MAP_ENTRY(mozilla::net::CacheFileIOListener)
   NS_INTERFACE_MAP_ENTRY(nsISupports)
 NS_INTERFACE_MAP_END_THREADSAFE
 
-CacheFileChunk::CacheFileChunk(CacheFile *aFile, uint32_t aIndex)
+CacheFileChunk::CacheFileChunk(CacheFile *aFile, uint32_t aIndex,
+                               bool aInitByWriter)
   : CacheMemoryConsumer(aFile->mOpenAsMemoryOnly ? MEMORY_ONLY : DONT_REPORT)
   , mIndex(aIndex)
   , mState(INITIAL)
   , mStatus(NS_OK)
   , mIsDirty(false)
   , mActiveChunk(false)
   , mDataSize(0)
+  , mReportedAllocation(0)
+  , mLimitAllocation(!aFile->mOpenAsMemoryOnly && aInitByWriter)
+  , mIsPriority(aFile->mPriority)
   , mBuf(nullptr)
   , mBufSize(0)
   , mRWBuf(nullptr)
   , mRWBufSize(0)
   , mReadHash(0)
   , mFile(aFile)
 {
-  LOG(("CacheFileChunk::CacheFileChunk() [this=%p]", this));
+  LOG(("CacheFileChunk::CacheFileChunk() [this=%p, index=%u, initByWriter=%d]",
+       this, aIndex, aInitByWriter));
   MOZ_COUNT_CTOR(CacheFileChunk);
 }
 
 CacheFileChunk::~CacheFileChunk()
 {
   LOG(("CacheFileChunk::~CacheFileChunk() [this=%p]", this));
   MOZ_COUNT_DTOR(CacheFileChunk);
 
   if (mBuf) {
     free(mBuf);
     mBuf = nullptr;
     mBufSize = 0;
+    ChunkAllocationChanged();
   }
 
   if (mRWBuf) {
     free(mRWBuf);
     mRWBuf = nullptr;
     mRWBufSize = 0;
+    ChunkAllocationChanged();
   }
 }
 
 void
-CacheFileChunk::InitNew(CacheFileChunkListener *aCallback)
+CacheFileChunk::InitNew()
 {
   mFile->AssertOwnsLock();
 
-  LOG(("CacheFileChunk::InitNew() [this=%p, listener=%p]", this, aCallback));
+  LOG(("CacheFileChunk::InitNew() [this=%p]", this));
 
   MOZ_ASSERT(mState == INITIAL);
+  MOZ_ASSERT(NS_SUCCEEDED(mStatus));
   MOZ_ASSERT(!mBuf);
   MOZ_ASSERT(!mRWBuf);
+  MOZ_ASSERT(!mIsDirty);
+  MOZ_ASSERT(mDataSize == 0);
 
-  mBuf = static_cast<char *>(moz_xmalloc(kMinBufSize));
-  mBufSize = kMinBufSize;
-  mDataSize = 0;
   mState = READY;
-  mIsDirty = true;
-
-  DoMemoryReport(MemorySize());
 }
 
 nsresult
 CacheFileChunk::Read(CacheFileHandle *aHandle, uint32_t aLen,
                      CacheHash::Hash16_t aHash,
                      CacheFileChunkListener *aCallback)
 {
   mFile->AssertOwnsLock();
 
   LOG(("CacheFileChunk::Read() [this=%p, handle=%p, len=%d, listener=%p]",
        this, aHandle, aLen, aCallback));
 
   MOZ_ASSERT(mState == INITIAL);
+  MOZ_ASSERT(NS_SUCCEEDED(mStatus));
   MOZ_ASSERT(!mBuf);
   MOZ_ASSERT(!mRWBuf);
   MOZ_ASSERT(aLen);
 
   nsresult rv;
 
-  mRWBuf = static_cast<char *>(moz_xmalloc(aLen));
-  mRWBufSize = aLen;
+  mState = READING;
+
+  if (CanAllocate(aLen)) {
+    mRWBuf = static_cast<char *>(moz_malloc(aLen));
+    if (mRWBuf) {
+      mRWBufSize = aLen;
+      ChunkAllocationChanged();
+    }
+  }
+
+  if (!mRWBuf) {
+    // Allocation was denied or failed
+    SetError(NS_ERROR_OUT_OF_MEMORY);
+    return mStatus;
+  }
 
   DoMemoryReport(MemorySize());
 
   rv = CacheFileIOManager::Read(aHandle, mIndex * kChunkSize, mRWBuf, aLen,
                                 true, this);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     rv = mIndex ? NS_ERROR_FILE_CORRUPTED : NS_ERROR_FILE_NOT_FOUND;
     SetError(rv);
   } else {
-    mState = READING;
     mListener = aCallback;
     mDataSize = aLen;
     mReadHash = aHash;
   }
 
   return rv;
 }
 
@@ -200,33 +217,34 @@ CacheFileChunk::Write(CacheFileHandle *a
                       CacheFileChunkListener *aCallback)
 {
   mFile->AssertOwnsLock();
 
   LOG(("CacheFileChunk::Write() [this=%p, handle=%p, listener=%p]",
        this, aHandle, aCallback));
 
   MOZ_ASSERT(mState == READY);
+  MOZ_ASSERT(NS_SUCCEEDED(mStatus));
   MOZ_ASSERT(!mRWBuf);
   MOZ_ASSERT(mBuf);
   MOZ_ASSERT(mDataSize); // Don't write chunk when it is empty
 
   nsresult rv;
 
+  mState = WRITING;
   mRWBuf = mBuf;
   mRWBufSize = mBufSize;
   mBuf = nullptr;
   mBufSize = 0;
 
   rv = CacheFileIOManager::Write(aHandle, mIndex * kChunkSize, mRWBuf,
                                  mDataSize, false, this);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     SetError(rv);
   } else {
-    mState = WRITING;
     mListener = aCallback;
     mIsDirty = false;
   }
 
   return rv;
 }
 
 void
@@ -319,21 +337,20 @@ CacheFileChunk::Index()
   return mIndex;
 }
 
 CacheHash::Hash16_t
 CacheFileChunk::Hash()
 {
   mFile->AssertOwnsLock();
 
-  MOZ_ASSERT(mBuf);
   MOZ_ASSERT(!mListener);
   MOZ_ASSERT(IsReady());
 
-  return CacheHash::Hash16(BufForReading(), mDataSize);
+  return CacheHash::Hash16(mDataSize ? BufForReading() : nullptr, mDataSize);
 }
 
 uint32_t
 CacheFileChunk::DataSize()
 {
   mFile->AssertOwnsLock();
   return mDataSize;
 }
@@ -344,17 +361,17 @@ CacheFileChunk::UpdateDataSize(uint32_t 
   mFile->AssertOwnsLock();
 
   MOZ_ASSERT(!aEOF, "Implement me! What to do with opened streams?");
   MOZ_ASSERT(aOffset <= mDataSize);
   MOZ_ASSERT(aLen != 0);
 
   // UpdateDataSize() is called only when we've written some data to the chunk
   // and we never write data anymore once some error occurs.
-  MOZ_ASSERT(mState != ERROR);
+  MOZ_ASSERT(NS_SUCCEEDED(mStatus));
 
   LOG(("CacheFileChunk::UpdateDataSize() [this=%p, offset=%d, len=%d, EOF=%d]",
        this, aOffset, aLen, aEOF));
 
   mIsDirty = true;
 
   int64_t fileSize = kChunkSize * mIndex + aOffset + aLen;
   bool notify = false;
@@ -408,29 +425,32 @@ CacheFileChunk::OnDataWritten(CacheFileH
   {
     CacheFileAutoLock lock(mFile);
 
     MOZ_ASSERT(mState == WRITING);
     MOZ_ASSERT(mListener);
 
     if (NS_WARN_IF(NS_FAILED(aResult))) {
       SetError(aResult);
-    } else {
-      mState = READY;
     }
 
+    mState = READY;
+
     if (!mBuf) {
       mBuf = mRWBuf;
       mBufSize = mRWBufSize;
+      mRWBuf = nullptr;
+      mRWBufSize = 0;
     } else {
       free(mRWBuf);
+      mRWBuf = nullptr;
+      mRWBufSize = 0;
+      ChunkAllocationChanged();
     }
 
-    mRWBuf = nullptr;
-    mRWBufSize = 0;
 
     DoMemoryReport(MemorySize());
 
     mListener.swap(listener);
   }
 
   listener->OnChunkWritten(aResult, this);
 
@@ -468,47 +488,79 @@ CacheFileChunk::OnDataRead(CacheFileHand
           mBufSize = mRWBufSize;
           mRWBuf = nullptr;
           mRWBufSize = 0;
         } else {
           LOG(("CacheFileChunk::OnDataRead() - Merging buffers. [this=%p]",
                this));
 
           // Merge data with write buffer
-          if (mRWBufSize < mBufSize) {
-            mRWBuf = static_cast<char *>(moz_xrealloc(mRWBuf, mBufSize));
-            mRWBufSize = mBufSize;
-          }
+          if (mRWBufSize >= mBufSize) {
+            // The new data will fit into the buffer that contains data read
+            // from the disk. Simply copy the valid pieces.
+            mValidityMap.Log();
+            for (uint32_t i = 0; i < mValidityMap.Length(); i++) {
+              if (mValidityMap[i].Offset() + mValidityMap[i].Len() > mBufSize) {
+                MOZ_CRASH("Unexpected error in validity map!");
+              }
+              memcpy(mRWBuf + mValidityMap[i].Offset(),
+                     mBuf + mValidityMap[i].Offset(), mValidityMap[i].Len());
+            }
+            mValidityMap.Clear();
 
-          mValidityMap.Log();
-          for (uint32_t i = 0 ; i < mValidityMap.Length() ; i++) {
-            memcpy(mRWBuf + mValidityMap[i].Offset(),
-                   mBuf + mValidityMap[i].Offset(), mValidityMap[i].Len());
+            free(mBuf);
+            mBuf = mRWBuf;
+            mBufSize = mRWBufSize;
+            mRWBuf = nullptr;
+            mRWBufSize = 0;
+            ChunkAllocationChanged();
+          } else {
+            // Buffer holding the new data is larger. Use it as the destination
+            // buffer to avoid reallocating mRWBuf. We need to copy those pieces
+            // from mRWBuf which are not valid in mBuf.
+            uint32_t invalidOffset = 0;
+            uint32_t invalidLength;
+            mValidityMap.Log();
+            for (uint32_t i = 0; i < mValidityMap.Length(); i++) {
+              MOZ_ASSERT(invalidOffset <= mValidityMap[i].Offset());
+              invalidLength = mValidityMap[i].Offset() - invalidOffset;
+              if (invalidLength > 0) {
+                if (invalidOffset + invalidLength > mRWBufSize) {
+                  MOZ_CRASH("Unexpected error in validity map!");
+                }
+                memcpy(mBuf + invalidOffset, mRWBuf + invalidOffset,
+                       invalidLength);
+              }
+              invalidOffset = mValidityMap[i].Offset() + mValidityMap[i].Len();
+            }
+            if (invalidOffset < mRWBufSize) {
+              invalidLength = invalidOffset - mRWBufSize;
+              memcpy(mBuf + invalidOffset, mRWBuf + invalidOffset,
+                     invalidLength);
+            }
+            mValidityMap.Clear();
+
+            free(mRWBuf);
+            mRWBuf = nullptr;
+            mRWBufSize = 0;
+            ChunkAllocationChanged();
           }
-          mValidityMap.Clear();
-
-          free(mBuf);
-          mBuf = mRWBuf;
-          mBufSize = mRWBufSize;
-          mRWBuf = nullptr;
-          mRWBufSize = 0;
 
           DoMemoryReport(MemorySize());
         }
       }
     }
 
     if (NS_FAILED(aResult)) {
       aResult = mIndex ? NS_ERROR_FILE_CORRUPTED : NS_ERROR_FILE_NOT_FOUND;
       SetError(aResult);
       mDataSize = 0;
-    } else {
-      mState = READY;
     }
 
+    mState = READY;
     mListener.swap(listener);
   }
 
   listener->OnChunkRead(aResult, this);
 
   return NS_OK;
 }
 
@@ -555,32 +607,34 @@ CacheFileChunk::GetStatus()
   mFile->AssertOwnsLock();
 
   return mStatus;
 }
 
 void
 CacheFileChunk::SetError(nsresult aStatus)
 {
-  if (NS_SUCCEEDED(mStatus)) {
-    MOZ_ASSERT(mState != ERROR);
-    mStatus = aStatus;
-    mState = ERROR;
-  } else {
-    MOZ_ASSERT(mState == ERROR);
+  MOZ_ASSERT(NS_FAILED(aStatus));
+
+  if (NS_FAILED(mStatus)) {
+    // Remember only the first error code.
+    return;
   }
+
+  mStatus = aStatus;
 }
 
 char *
 CacheFileChunk::BufForWriting() const
 {
   mFile->AssertOwnsLock();
 
   MOZ_ASSERT(mBuf); // Writer should always first call EnsureBufSize()
 
+  MOZ_ASSERT(NS_SUCCEEDED(mStatus));
   MOZ_ASSERT((mState == READY && !mRWBuf) ||
              (mState == WRITING && mRWBuf) ||
              (mState == READING && mRWBuf));
 
   return mBuf;
 }
 
 const char *
@@ -589,27 +643,28 @@ CacheFileChunk::BufForReading() const
   mFile->AssertOwnsLock();
 
   MOZ_ASSERT((mState == READY && mBuf && !mRWBuf) ||
              (mState == WRITING && mRWBuf));
 
   return mBuf ? mBuf : mRWBuf;
 }
 
-void
+MOZ_WARN_UNUSED_RESULT nsresult
 CacheFileChunk::EnsureBufSize(uint32_t aBufSize)
 {
   mFile->AssertOwnsLock();
 
   // EnsureBufSize() is called only when we want to write some data to the chunk
   // and we never write data anymore once some error occurs.
-  MOZ_ASSERT(mState != ERROR);
+  MOZ_ASSERT(NS_SUCCEEDED(mStatus));
 
-  if (mBufSize >= aBufSize)
-    return;
+  if (mBufSize >= aBufSize) {
+    return NS_OK;
+  }
 
   bool copy = false;
   if (!mBuf && mState == WRITING) {
     // We need to duplicate the data that is being written on the background
     // thread, so make sure that all the data fits into the new buffer.
     copy = true;
 
     if (mRWBufSize > aBufSize)
@@ -624,23 +679,37 @@ CacheFileChunk::EnsureBufSize(uint32_t a
   aBufSize |= aBufSize >> 8;
   aBufSize |= aBufSize >> 16;
   aBufSize++;
 
   const uint32_t minBufSize = kMinBufSize;
   const uint32_t maxBufSize = kChunkSize;
   aBufSize = clamped(aBufSize, minBufSize, maxBufSize);
 
-  mBuf = static_cast<char *>(moz_xrealloc(mBuf, aBufSize));
+  if (!CanAllocate(aBufSize - mBufSize)) {
+    SetError(NS_ERROR_OUT_OF_MEMORY);
+    return mStatus;
+  }
+
+  char *newBuf = static_cast<char *>(moz_realloc(mBuf, aBufSize));
+  if (!newBuf) {
+    SetError(NS_ERROR_OUT_OF_MEMORY);
+    return mStatus;
+  }
+
+  mBuf = newBuf;
   mBufSize = aBufSize;
+  ChunkAllocationChanged();
 
   if (copy)
     memcpy(mBuf, mRWBuf, mRWBufSize);
 
   DoMemoryReport(MemorySize());
+
+  return NS_OK;
 }
 
 // Memory reporting
 
 size_t
 CacheFileChunk::SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const
 {
   size_t n = 0;
@@ -652,10 +721,55 @@ CacheFileChunk::SizeOfExcludingThis(mozi
 }
 
 size_t
 CacheFileChunk::SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const
 {
   return mallocSizeOf(this) + SizeOfExcludingThis(mallocSizeOf);
 }
 
+bool
+CacheFileChunk::CanAllocate(uint32_t aSize)
+{
+  if (!mLimitAllocation) {
+    return true;
+  }
+
+  LOG(("CacheFileChunk::CanAllocate() [this=%p, size=%u]", this, aSize));
+
+  uint32_t limit = CacheObserver::MaxDiskChunksMemoryUsage(mIsPriority);
+  if (limit == 0) {
+    return true;
+  }
+
+  uint32_t usage = ChunksMemoryUsage();
+  if (usage + aSize > limit) {
+    LOG(("CacheFileChunk::CanAllocate() - Returning false. [this=%p]", this));
+    return false;
+  }
+
+  return true;
+}
+
+void
+CacheFileChunk::ChunkAllocationChanged()
+{
+  if (!mLimitAllocation) {
+    return;
+  }
+
+  ChunksMemoryUsage() -= mReportedAllocation;
+  mReportedAllocation = mBufSize + mRWBufSize;
+  ChunksMemoryUsage() += mReportedAllocation;
+  LOG(("CacheFileChunk::ChunkAllocationChanged() - %s chunks usage %u "
+       "[this=%p]", mIsPriority ? "Priority" : "Normal",
+       static_cast<uint32_t>(ChunksMemoryUsage()), this));
+}
+
+mozilla::Atomic<uint32_t>& CacheFileChunk::ChunksMemoryUsage()
+{
+  static mozilla::Atomic<uint32_t> chunksMemoryUsage(0);
+  static mozilla::Atomic<uint32_t> prioChunksMemoryUsage(0);
+  return mIsPriority ? prioChunksMemoryUsage : chunksMemoryUsage;
+}
+
 } // net
 } // mozilla
--- a/netwerk/cache2/CacheFileChunk.h
+++ b/netwerk/cache2/CacheFileChunk.h
@@ -65,19 +65,19 @@ public:
 
 class CacheFileChunk : public CacheFileIOListener
                      , public CacheMemoryConsumer
 {
 public:
   NS_DECL_THREADSAFE_ISUPPORTS
   bool DispatchRelease();
 
-  CacheFileChunk(CacheFile *aFile, uint32_t aIndex);
+  CacheFileChunk(CacheFile *aFile, uint32_t aIndex, bool aInitByWriter);
 
-  void     InitNew(CacheFileChunkListener *aCallback);
+  void     InitNew();
   nsresult Read(CacheFileHandle *aHandle, uint32_t aLen,
                 CacheHash::Hash16_t aHash,
                 CacheFileChunkListener *aCallback);
   nsresult Write(CacheFileHandle *aHandle, CacheFileChunkListener *aCallback);
   void     WaitForUpdate(CacheFileChunkListener *aCallback);
   nsresult CancelWait(CacheFileChunkListener *aCallback);
   nsresult NotifyUpdateListeners();
 
@@ -98,48 +98,56 @@ public:
   bool   IsReady() const;
   bool   IsDirty() const;
 
   nsresult GetStatus();
   void     SetError(nsresult aStatus);
 
   char *       BufForWriting() const;
   const char * BufForReading() const;
-  void         EnsureBufSize(uint32_t aBufSize);
+  nsresult     EnsureBufSize(uint32_t aBufSize);
   uint32_t     MemorySize() const { return sizeof(CacheFileChunk) + mRWBufSize + mBufSize; }
 
   // Memory reporting
   size_t SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
   size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
 
 private:
   friend class CacheFileInputStream;
   friend class CacheFileOutputStream;
   friend class CacheFile;
 
   virtual ~CacheFileChunk();
 
+  bool CanAllocate(uint32_t aSize);
+  void ChunkAllocationChanged();
+  mozilla::Atomic<uint32_t>& ChunksMemoryUsage();
+
   enum EState {
     INITIAL = 0,
     READING = 1,
     WRITING = 2,
-    READY   = 3,
-    ERROR   = 4
+    READY   = 3
   };
 
   uint32_t mIndex;
   EState   mState;
   nsresult mStatus;
   bool     mIsDirty;
   bool     mActiveChunk; // Is true iff the chunk is in CacheFile::mChunks.
                          // Adding/removing chunk to/from mChunks as well as
                          // changing this member happens under the CacheFile's
                          // lock.
   uint32_t mDataSize;
 
+  uint32_t   mReportedAllocation;
+  bool const mLimitAllocation : 1; // Whether this chunk respects limit for disk
+                                   // chunks memory usage.
+  bool const mIsPriority : 1;
+
   char    *mBuf;
   uint32_t mBufSize;
 
   char               *mRWBuf;
   uint32_t            mRWBufSize;
   CacheHash::Hash16_t mReadHash;
 
   nsRefPtr<CacheFile>              mFile; // is null if chunk is cached to
--- a/netwerk/cache2/CacheFileInputStream.cpp
+++ b/netwerk/cache2/CacheFileInputStream.cpp
@@ -23,17 +23,17 @@ CacheFileInputStream::Release()
 
   if (0 == count) {
     mRefCnt = 1;
     delete (this);
     return 0;
   }
 
   if (count == 1) {
-    mFile->RemoveInput(this);
+    mFile->RemoveInput(this, mStatus);
   }
 
   return count;
 }
 
 NS_INTERFACE_MAP_BEGIN(CacheFileInputStream)
   NS_INTERFACE_MAP_ENTRY(nsIInputStream)
   NS_INTERFACE_MAP_ENTRY(nsIAsyncInputStream)
@@ -148,23 +148,25 @@ CacheFileInputStream::ReadSegments(nsWri
       else {
         return NS_BASE_STREAM_WOULD_BLOCK;
       }
     }
 
     int64_t canRead;
     const char *buf;
     CanRead(&canRead, &buf);
+    if (NS_FAILED(mStatus)) {
+      return mStatus;
+    }
 
     if (canRead < 0) {
       // file was truncated ???
       MOZ_ASSERT(false, "SetEOF is currenty not implemented?!");
       rv = NS_OK;
-    }
-    else if (canRead > 0) {
+    } else if (canRead > 0) {
       uint32_t toRead = std::min(static_cast<uint32_t>(canRead), aCount);
 
       // We need to release the lock to avoid lock re-entering unless the
       // caller is Read() method.
 #ifdef DEBUG
       int64_t oldPos = mPos;
 #endif
       mInReadSegments = true;
@@ -194,18 +196,17 @@ CacheFileInputStream::ReadSegments(nsWri
 
         if (mChunk && aCount) {
           // We have the next chunk! Go on.
           continue;
         }
       }
 
       rv = NS_OK;
-    }
-    else {
+    } else {
       if (mFile->mOutput)
         rv = NS_BASE_STREAM_WOULD_BLOCK;
       else {
         rv = NS_OK;
       }
     }
 
     break;
@@ -537,17 +538,24 @@ CacheFileInputStream::CanRead(int64_t *a
 {
   mFile->AssertOwnsLock();
 
   MOZ_ASSERT(mChunk);
   MOZ_ASSERT(mPos / kChunkSize == mChunk->Index());
 
   uint32_t chunkOffset = mPos - (mPos / kChunkSize) * kChunkSize;
   *aCanRead = mChunk->DataSize() - chunkOffset;
-  *aBuf = mChunk->BufForReading() + chunkOffset;
+  if (*aCanRead > 0) {
+    *aBuf = mChunk->BufForReading() + chunkOffset;
+  } else {
+    *aBuf = nullptr;
+    if (NS_FAILED(mChunk->GetStatus())) {
+      CloseWithStatusLocked(mChunk->GetStatus());
+    }
+  }
 
   LOG(("CacheFileInputStream::CanRead() [this=%p, canRead=%lld]",
        this, *aCanRead));
 }
 
 void
 CacheFileInputStream::NotifyListener()
 {
@@ -598,16 +606,22 @@ CacheFileInputStream::MaybeNotifyListene
   MOZ_ASSERT(mPos / kChunkSize == mChunk->Index());
 
   if (mWaitingForUpdate)
     return;
 
   int64_t canRead;
   const char *buf;
   CanRead(&canRead, &buf);
+  if (NS_FAILED(mStatus)) {
+    // CanRead() called CloseWithStatusLocked() which called
+    // MaybeNotifyListener() so the listener was already notified. Stop here.
+    MOZ_ASSERT(!mCallback);
+    return;
+  }
 
   if (canRead > 0) {
     if (!(mCallbackFlags & WAIT_CLOSURE_ONLY))
       NotifyListener();
   }
   else if (canRead == 0) {
     if (!mFile->mOutput) {
       // EOF
--- a/netwerk/cache2/CacheFileInputStream.h
+++ b/netwerk/cache2/CacheFileInputStream.h
@@ -41,16 +41,19 @@ public:
   uint32_t GetPosition() const { return mPos; };
 
 private:
   virtual ~CacheFileInputStream();
 
   nsresult CloseWithStatusLocked(nsresult aStatus);
   void ReleaseChunk();
   void EnsureCorrectChunk(bool aReleaseOnly);
+
+  // CanRead returns negative value when output stream truncates the data before
+  // the input stream's mPos.
   void CanRead(int64_t *aCanRead, const char **aBuf);
   void NotifyListener();
   void MaybeNotifyListener();
 
   nsRefPtr<CacheFile>      mFile;
   nsRefPtr<CacheFileChunk> mChunk;
   int64_t                  mPos;
   bool                     mClosed;
--- a/netwerk/cache2/CacheFileOutputStream.cpp
+++ b/netwerk/cache2/CacheFileOutputStream.cpp
@@ -22,17 +22,17 @@ CacheFileOutputStream::Release()
   NS_PRECONDITION(0 != mRefCnt, "dup release");
   nsrefcnt count = --mRefCnt;
   NS_LOG_RELEASE(this, count, "CacheFileOutputStream");
 
   if (0 == count) {
     mRefCnt = 1;
     {
       CacheFileAutoLock lock(mFile);
-      mFile->RemoveOutput(this);
+      mFile->RemoveOutput(this, mStatus);
     }
     delete (this);
     return 0;
   }
 
   return count;
 }
 
@@ -102,25 +102,33 @@ CacheFileOutputStream::Write(const char 
     CloseWithStatusLocked(NS_ERROR_FILE_TOO_BIG);
     return NS_ERROR_FILE_TOO_BIG;
   }
 
   *_retval = aCount;
 
   while (aCount) {
     EnsureCorrectChunk(false);
-    if (NS_FAILED(mStatus))
+    if (NS_FAILED(mStatus)) {
       return mStatus;
+    }
 
     FillHole();
+    if (NS_FAILED(mStatus)) {
+      return mStatus;
+    }
 
     uint32_t chunkOffset = mPos - (mPos / kChunkSize) * kChunkSize;
     uint32_t canWrite = kChunkSize - chunkOffset;
     uint32_t thisWrite = std::min(static_cast<uint32_t>(canWrite), aCount);
-    mChunk->EnsureBufSize(chunkOffset + thisWrite);
+    nsresult rv = mChunk->EnsureBufSize(chunkOffset + thisWrite);
+    if (NS_FAILED(rv)) {
+      CloseWithStatusLocked(rv);
+      return rv;
+    }
     memcpy(mChunk->BufForWriting() + chunkOffset, aBuf, thisWrite);
 
     mPos += thisWrite;
     aBuf += thisWrite;
     aCount -= thisWrite;
 
     mChunk->UpdateDataSize(chunkOffset, thisWrite, false);
   }
@@ -189,17 +197,17 @@ CacheFileOutputStream::CloseWithStatusLo
   if (mChunk) {
     ReleaseChunk();
   }
 
   if (mCallback) {
     NotifyListener();
   }
 
-  mFile->RemoveOutput(this);
+  mFile->RemoveOutput(this, mStatus);
 
   return NS_OK;
 }
 
 NS_IMETHODIMP
 CacheFileOutputStream::AsyncWait(nsIOutputStreamCallback *aCallback,
                                  uint32_t aFlags,
                                  uint32_t aRequestedCount,
@@ -385,17 +393,22 @@ CacheFileOutputStream::FillHole()
 
   uint32_t pos = mPos - (mPos / kChunkSize) * kChunkSize;
   if (mChunk->DataSize() >= pos)
     return;
 
   LOG(("CacheFileOutputStream::FillHole() - Zeroing hole in chunk %d, range "
        "%d-%d [this=%p]", mChunk->Index(), mChunk->DataSize(), pos - 1, this));
 
-  mChunk->EnsureBufSize(pos);
+  nsresult rv = mChunk->EnsureBufSize(pos);
+  if (NS_FAILED(rv)) {
+    CloseWithStatusLocked(rv);
+    return;
+  }
+
   memset(mChunk->BufForWriting() + mChunk->DataSize(), 0,
          pos - mChunk->DataSize());
 
   mChunk->UpdateDataSize(mChunk->DataSize(), pos - mChunk->DataSize(), false);
 }
 
 void
 CacheFileOutputStream::NotifyListener()
--- a/netwerk/cache2/CacheIndex.cpp
+++ b/netwerk/cache2/CacheIndex.cpp
@@ -2278,17 +2278,17 @@ CacheIndex::MergeJournal()
 }
 
 // static
 PLDHashOperator
 CacheIndex::ProcessJournalEntry(CacheIndexEntry *aEntry, void* aClosure)
 {
   CacheIndex *index = static_cast<CacheIndex *>(aClosure);
 
-  LOG(("CacheFile::ProcessJournalEntry() [hash=%08x%08x%08x%08x%08x]",
+  LOG(("CacheIndex::ProcessJournalEntry() [hash=%08x%08x%08x%08x%08x]",
        LOGSHA1(aEntry->Hash())));
 
   CacheIndexEntry *entry = index->mIndex.GetEntry(*aEntry->Hash());
 
   CacheIndexEntryAutoManage emng(aEntry->Hash(), index);
 
   if (aEntry->IsRemoved()) {
     if (entry) {
--- a/netwerk/cache2/CacheObserver.cpp
+++ b/netwerk/cache2/CacheObserver.cpp
@@ -60,16 +60,22 @@ static uint32_t const kDefaultPreloadChu
 uint32_t CacheObserver::sPreloadChunkCount = kDefaultPreloadChunkCount;
 
 static uint32_t const kDefaultMaxMemoryEntrySize = 4 * 1024; // 4 MB
 uint32_t CacheObserver::sMaxMemoryEntrySize = kDefaultMaxMemoryEntrySize;
 
 static uint32_t const kDefaultMaxDiskEntrySize = 50 * 1024; // 50 MB
 uint32_t CacheObserver::sMaxDiskEntrySize = kDefaultMaxDiskEntrySize;
 
+static uint32_t const kDefaultMaxDiskChunksMemoryUsage = 10 * 1024; // 10MB
+uint32_t CacheObserver::sMaxDiskChunksMemoryUsage = kDefaultMaxDiskChunksMemoryUsage;
+
+static uint32_t const kDefaultMaxDiskPriorityChunksMemoryUsage = 10 * 1024; // 10MB
+uint32_t CacheObserver::sMaxDiskPriorityChunksMemoryUsage = kDefaultMaxDiskPriorityChunksMemoryUsage;
+
 static uint32_t const kDefaultCompressionLevel = 1;
 uint32_t CacheObserver::sCompressionLevel = kDefaultCompressionLevel;
 
 static bool kDefaultSanitizeOnShutdown = false;
 bool CacheObserver::sSanitizeOnShutdown = kDefaultSanitizeOnShutdown;
 
 static bool kDefaultClearCacheOnShutdown = false;
 bool CacheObserver::sClearCacheOnShutdown = kDefaultClearCacheOnShutdown;
@@ -147,16 +153,21 @@ CacheObserver::AttachToPreferences()
   mozilla::Preferences::AddUintVarCache(
     &sPreloadChunkCount, "browser.cache.disk.preload_chunk_count", kDefaultPreloadChunkCount);
 
   mozilla::Preferences::AddUintVarCache(
     &sMaxDiskEntrySize, "browser.cache.disk.max_entry_size", kDefaultMaxDiskEntrySize);
   mozilla::Preferences::AddUintVarCache(
     &sMaxMemoryEntrySize, "browser.cache.memory.max_entry_size", kDefaultMaxMemoryEntrySize);
 
+  mozilla::Preferences::AddUintVarCache(
+    &sMaxDiskChunksMemoryUsage, "browser.cache.disk.max_chunks_memory_usage", kDefaultMaxDiskChunksMemoryUsage);
+  mozilla::Preferences::AddUintVarCache(
+    &sMaxDiskPriorityChunksMemoryUsage, "browser.cache.disk.max_priority_chunks_memory_usage", kDefaultMaxDiskPriorityChunksMemoryUsage);
+
   // http://mxr.mozilla.org/mozilla-central/source/netwerk/cache/nsCacheEntryDescriptor.cpp#367
   mozilla::Preferences::AddUintVarCache(
     &sCompressionLevel, "browser.cache.compression_level", kDefaultCompressionLevel);
 
   mozilla::Preferences::GetComplex(
     "browser.cache.disk.parent_directory", NS_GET_IID(nsIFile),
     getter_AddRefs(mCacheParentDirectoryOverride));
 
--- a/netwerk/cache2/CacheObserver.h
+++ b/netwerk/cache2/CacheObserver.h
@@ -41,16 +41,19 @@ class CacheObserver : public nsIObserver
   static bool const SmartCacheSizeEnabled()
     { return sSmartCacheSizeEnabled; }
   static uint32_t const PreloadChunkCount()
     { return sPreloadChunkCount; }
   static uint32_t const MaxMemoryEntrySize() // result in bytes.
     { return sMaxMemoryEntrySize << 10; }
   static uint32_t const MaxDiskEntrySize() // result in bytes.
     { return sMaxDiskEntrySize << 10; }
+  static uint32_t const MaxDiskChunksMemoryUsage(bool aPriority) // result in bytes.
+    { return aPriority ? sMaxDiskPriorityChunksMemoryUsage << 10
+                       : sMaxDiskChunksMemoryUsage << 10; }
   static uint32_t const CompressionLevel()
     { return sCompressionLevel; }
   static uint32_t const HalfLifeSeconds()
     { return sHalfLifeHours * 60 * 60; }
   static int32_t const HalfLifeExperiment()
     { return sHalfLifeExperiment; }
   static bool const ClearCacheOnShutdown()
     { return sSanitizeOnShutdown && sClearCacheOnShutdown; }
@@ -70,16 +73,18 @@ private:
   static uint32_t sMetadataMemoryLimit;
   static int32_t sMemoryCacheCapacity;
   static int32_t sAutoMemoryCacheCapacity;
   static uint32_t sDiskCacheCapacity;
   static bool sSmartCacheSizeEnabled;
   static uint32_t sPreloadChunkCount;
   static uint32_t sMaxMemoryEntrySize;
   static uint32_t sMaxDiskEntrySize;
+  static uint32_t sMaxDiskChunksMemoryUsage;
+  static uint32_t sMaxDiskPriorityChunksMemoryUsage;
   static uint32_t sCompressionLevel;
   static uint32_t sHalfLifeHours;
   static int32_t sHalfLifeExperiment;
   static bool sSanitizeOnShutdown;
   static bool sClearCacheOnShutdown;
 
   // Non static properties, accessible via sSelf
   nsCOMPtr<nsIFile> mCacheParentDirectoryOverride;
--- a/netwerk/protocol/http/nsHttpChannel.cpp
+++ b/netwerk/protocol/http/nsHttpChannel.cpp
@@ -71,19 +71,20 @@ namespace mozilla { namespace net {
 
 namespace {
 
 // True if the local cache should be bypassed when processing a request.
 #define BYPASS_LOCAL_CACHE(loadFlags) \
         (loadFlags & (nsIRequest::LOAD_BYPASS_CACHE | \
                       nsICachingChannel::LOAD_BYPASS_LOCAL_CACHE))
 
-#define CACHE_FILE_GONE(result) \
+#define RECOVER_FROM_CACHE_FILE_ERROR(result) \
         ((result) == NS_ERROR_FILE_NOT_FOUND || \
-         (result) == NS_ERROR_FILE_CORRUPTED)
+         (result) == NS_ERROR_FILE_CORRUPTED || \
+         (result) == NS_ERROR_OUT_OF_MEMORY)
 
 static NS_DEFINE_CID(kStreamListenerTeeCID, NS_STREAMLISTENERTEE_CID);
 static NS_DEFINE_CID(kStreamTransportServiceCID,
                      NS_STREAMTRANSPORTSERVICE_CID);
 
 enum CacheDisposition {
     kCacheHit = 1,
     kCacheHitViaReval = 2,
@@ -4922,19 +4923,20 @@ nsHttpChannel::OnStartRequest(nsIRequest
         // the response head may be null if the transaction was cancelled.  in
         // which case we just need to call OnStartRequest/OnStopRequest.
         if (mResponseHead)
             return ProcessResponse();
 
         NS_WARNING("No response head in OnStartRequest");
     }
 
-    // cache file could be deleted on our behalf, reload from network here.
-    if (mCacheEntry && mCachePump && CACHE_FILE_GONE(mStatus)) {
-        LOG(("  cache file gone, reloading from server"));
+    // cache file could be deleted on our behalf, it could contain errors or
+    // it failed to allocate memory, reload from network here.
+    if (mCacheEntry && mCachePump && RECOVER_FROM_CACHE_FILE_ERROR(mStatus)) {
+        LOG(("  cache file error, reloading from server"));
         mCacheEntry->AsyncDoom(nullptr);
         nsresult rv = StartRedirectChannelToURI(mURI, nsIChannelEventSink::REDIRECT_INTERNAL);
         if (NS_SUCCEEDED(rv))
             return NS_OK;
     }
 
     // avoid crashing if mListener happens to be null...
     if (!mListener) {
new file mode 100644
--- /dev/null
+++ b/netwerk/test/unit/test_cache2-25-chunk-memory-limit.js
@@ -0,0 +1,51 @@
+Components.utils.import('resource://gre/modules/LoadContextInfo.jsm');
+
+function gen_200k()
+{
+  var i;
+  var data="0123456789ABCDEFGHIJLKMNO";
+  for (i=0; i<13; i++)
+    data+=data;
+  return data;
+}
+
+// Keep the output stream of the first entry in a global variable, so the
+// CacheFile and its buffer isn't released before we write the data to the
+// second entry.
+var oStr;
+
+function run_test()
+{
+  do_get_profile();
+
+  if (!newCacheBackEndUsed()) {
+    do_check_true(true, "This test doesn't run when the old cache back end is used since the behavior is different");
+    return;
+  }
+
+  var prefBranch = Cc["@mozilla.org/preferences-service;1"].
+                     getService(Ci.nsIPrefBranch);
+
+  // set max chunks memory so that only one full chunk fits within the limit
+  prefBranch.setIntPref("browser.cache.disk.max_chunks_memory_usage", 300);
+
+  asyncOpenCacheEntry("http://a/", "disk", Ci.nsICacheStorage.OPEN_NORMALLY, null,
+    function(status, entry) {
+      do_check_eq(status, Cr.NS_OK);
+      oStr = entry.openOutputStream(0);
+      var data = gen_200k();
+      do_check_eq(data.length, oStr.write(data, data.length));
+
+      asyncOpenCacheEntry("http://b/", "disk", Ci.nsICacheStorage.OPEN_NORMALLY, null,
+        function(status, entry) {
+          do_check_eq(status, Cr.NS_OK);
+          var oStr2 = entry.openOutputStream(0);
+          do_check_throws_nsIException(() => oStr2.write(data, data.length), 'NS_ERROR_OUT_OF_MEMORY');
+          finish_cache2_test();
+        }
+      );
+    }
+  );
+
+  do_test_pending();
+}
--- a/netwerk/test/unit/xpcshell.ini
+++ b/netwerk/test/unit/xpcshell.ini
@@ -59,16 +59,17 @@ skip-if = os == "android"
 [test_cache2-19-range-206.js]
 [test_cache2-20-range-200.js]
 [test_cache2-21-anon-storage.js]
 [test_cache2-22-anon-visit.js]
 [test_cache2-23-read-over-chunk.js]
 [test_cache2-24-exists.js]
 # Bug 675039, comment 6: "The difference is that the memory cache is disabled in Armv6 builds."
 skip-if = os == "android"
+[test_cache2-25-chunk-memory-limit.js]
 [test_cache2-26-no-outputstream-open.js]
 # GC, that this patch is depenedent on, doesn't work well on Android."
 skip-if = os == "android"
 [test_304_responses.js]
 # Bug 675039: test hangs on Android-armv6 
 skip-if = os == "android"
 [test_cacheForOfflineUse_no-store.js]
 [test_307_redirect.js]
--- a/toolkit/components/telemetry/Histograms.json
+++ b/toolkit/components/telemetry/Histograms.json
@@ -6173,16 +6173,28 @@
   "NETWORK_CACHE_V1_HIT_TIME_MS": {
     "expires_in_version": "never",
     "kind": "exponential",
     "high": "10000",
     "n_buckets": 50,
     "extended_statistics_ok": true,
     "description": "Time spent to open an existing cache entry"
   },
+  "NETWORK_CACHE_V2_OUTPUT_STREAM_STATUS": {
+    "expires_in_version": "never",
+    "kind": "enumerated",
+    "n_values": "7",
+    "description": "Final status of the CacheFileOutputStream (0=ok, 1=other error, 2=out of memory, 3=disk full, 4=file corrupted, 5=file not found, 6=binding aborted)"
+  },
+  "NETWORK_CACHE_V2_INPUT_STREAM_STATUS": {
+    "expires_in_version": "never",
+    "kind": "enumerated",
+    "n_values": "7",
+    "description": "Final status of the CacheFileInputStream (0=ok, 1=other error, 2=out of memory, 3=disk full, 4=file corrupted, 5=file not found, 6=binding aborted)"
+  },
   "SQLITEBRIDGE_PROVIDER_PASSWORDS_LOCKED": {
     "expires_in_version": "never",
     "kind": "enumerated",
     "n_values": "10",
     "description": "The number of errors using the PasswordsProvider due to a locked DB."
   },
   "SQLITEBRIDGE_PROVIDER_FORMS_LOCKED": {
     "expires_in_version": "never",