Bug 1342722 - Firefox cache used all available inodes on the system, r=honzab
authorMichal Novotny <michal.novotny@gmail.com>
Mon, 20 Mar 2017 18:49:04 +0100
changeset 348491 474f08673e24ff21a161d4f3c6ce5b1623b528f8
parent 348490 0fe17d74ce9ea07f9ff0308b4c3619206e563fcb
child 348492 e7684ec77d0493315e7d19ec04ba9ab363d953d0
push id31526
push userkwierso@gmail.com
push dateTue, 21 Mar 2017 01:20:02 +0000
treeherdermozilla-central@5fe5dcf1c10a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewershonzab
bugs1342722
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1342722 - Firefox cache used all available inodes on the system, r=honzab When overlimit eviction is too slow the new content might be stored much faster than the old one is deleted. In such case the cache can grow far beyond the cache size limit. This patch implements a hard limit (105% of cache size limit) when we stop caching any new content.
netwerk/cache2/CacheFileIOManager.cpp
netwerk/cache2/CacheFileIOManager.h
--- a/netwerk/cache2/CacheFileIOManager.cpp
+++ b/netwerk/cache2/CacheFileIOManager.cpp
@@ -1085,16 +1085,17 @@ StaticRefPtr<CacheFileIOManager> CacheFi
 
 NS_IMPL_ISUPPORTS(CacheFileIOManager, nsITimerCallback)
 
 CacheFileIOManager::CacheFileIOManager()
   : mShuttingDown(false)
   , mTreeCreated(false)
   , mTreeCreationFailed(false)
   , mOverLimitEvicting(false)
+  , mCacheSizeOnHardLimit(false)
   , mRemovingTrashDirs(false)
 {
   LOG(("CacheFileIOManager::CacheFileIOManager [this=%p]", this));
   MOZ_ASSERT(!gInstance, "multiple CacheFileIOManager instances!");
 }
 
 CacheFileIOManager::~CacheFileIOManager()
 {
@@ -1986,18 +1987,25 @@ CacheFileIOManager::WriteInternal(CacheF
     NSPRHandleUsed(aHandle);
   }
 
   // Check again, OpenNSPRHandle could figure out the file was gone.
   if (!aHandle->mFileExists) {
     return NS_ERROR_NOT_AVAILABLE;
   }
 
-  // Check whether this write would cause critical low disk space.
+  // When this operation would increase cache size, check whether the cache size
+  // reached the hard limit and whether it would cause critical low disk space.
   if (aHandle->mFileSize < aOffset + aCount) {
+    if (mOverLimitEvicting && mCacheSizeOnHardLimit) {
+      LOG(("CacheFileIOManager::WriteInternal() - failing because cache size "
+           "reached hard limit!"));
+      return NS_ERROR_FILE_DISK_FULL;
+    }
+
     int64_t freeSpace = -1;
     rv = mCacheDirectory->GetDiskSpaceAvailable(&freeSpace);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       LOG(("CacheFileIOManager::WriteInternal() - GetDiskSpaceAvailable() "
            "failed! [rv=0x%08" PRIx32 "]", static_cast<uint32_t>(rv)));
     } else {
       uint32_t limit = CacheObserver::DiskFreeSpaceHardLimit();
       if (freeSpace - aOffset - aCount + aHandle->mFileSize < limit) {
@@ -2518,18 +2526,25 @@ CacheFileIOManager::TruncateSeekSetEOFIn
     NSPRHandleUsed(aHandle);
   }
 
   // Check again, OpenNSPRHandle could figure out the file was gone.
   if (!aHandle->mFileExists) {
     return NS_ERROR_NOT_AVAILABLE;
   }
 
-  // Check whether this operation would cause critical low disk space.
+  // When this operation would increase cache size, check whether the cache size
+  // reached the hard limit and whether it would cause critical low disk space.
   if (aHandle->mFileSize < aEOFPos) {
+    if (mOverLimitEvicting && mCacheSizeOnHardLimit) {
+      LOG(("CacheFileIOManager::TruncateSeekSetEOFInternal() - failing because "
+           "cache size reached hard limit!"));
+      return NS_ERROR_FILE_DISK_FULL;
+    }
+
     int64_t freeSpace = -1;
     rv = mCacheDirectory->GetDiskSpaceAvailable(&freeSpace);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       LOG(("CacheFileIOManager::TruncateSeekSetEOFInternal() - "
            "GetDiskSpaceAvailable() failed! [rv=0x%08" PRIx32 "]",
            static_cast<uint32_t>(rv)));
     } else {
       uint32_t limit = CacheObserver::DiskFreeSpaceHardLimit();
@@ -2784,25 +2799,41 @@ CacheFileIOManager::OverLimitEvictionInt
     NS_ENSURE_SUCCESS(rv, rv);
 
     uint32_t cacheLimit = CacheObserver::DiskCacheCapacity() >> 10;
     uint32_t freeSpaceLimit = CacheObserver::DiskFreeSpaceSoftLimit();
 
     if (cacheUsage > cacheLimit) {
       LOG(("CacheFileIOManager::OverLimitEvictionInternal() - Cache size over "
            "limit. [cacheSize=%u, limit=%u]", cacheUsage, cacheLimit));
+
+      // We allow cache size to go over the specified limit. Eviction should
+      // keep the size within the limit in a long run, but in case the eviction
+      // is too slow, the cache could go way over the limit. To prevent this we
+      // set flag mCacheSizeOnHardLimit when the size reaches 105% of the limit
+      // and WriteInternal() and TruncateSeekSetEOFInternal() fail to cache
+      // additional data.
+      if ((cacheUsage - cacheLimit) > (cacheLimit / 20)) {
+        LOG(("CacheFileIOManager::OverLimitEvictionInternal() - Cache size "
+             "reached hard limit."));
+        mCacheSizeOnHardLimit = true;
+      } else {
+        mCacheSizeOnHardLimit = false;
+      }
     } else if (freeSpace != 1 && freeSpace < freeSpaceLimit) {
       LOG(("CacheFileIOManager::OverLimitEvictionInternal() - Free space under "
            "limit. [freeSpace=%" PRId64 ", freeSpaceLimit=%u]", freeSpace,
            freeSpaceLimit));
     } else {
       LOG(("CacheFileIOManager::OverLimitEvictionInternal() - Cache size and "
            "free space in limits. [cacheSize=%ukB, cacheSizeLimit=%ukB, "
            "freeSpace=%" PRId64 ", freeSpaceLimit=%u]", cacheUsage, cacheLimit,
            freeSpace, freeSpaceLimit));
+
+      mCacheSizeOnHardLimit = false;
       return NS_OK;
     }
 
     if (CacheIOThread::YieldAndRerun()) {
       LOG(("CacheFileIOManager::OverLimitEvictionInternal() - Breaking loop "
            "for higher level events."));
       mOverLimitEvicting = true;
       return NS_OK;
--- a/netwerk/cache2/CacheFileIOManager.h
+++ b/netwerk/cache2/CacheFileIOManager.h
@@ -469,16 +469,20 @@ private:
   bool                                 mTreeCreated;
   bool                                 mTreeCreationFailed;
   CacheFileHandles                     mHandles;
   nsTArray<CacheFileHandle *>          mHandlesByLastUsed;
   nsTArray<CacheFileHandle *>          mSpecialHandles;
   nsTArray<RefPtr<CacheFile> >         mScheduledMetadataWrites;
   nsCOMPtr<nsITimer>                   mMetadataWritesTimer;
   bool                                 mOverLimitEvicting;
+  // When overlimit eviction is too slow and cache size reaches 105% of the
+  // limit, this flag is set and no other content is cached to prevent
+  // uncontrolled cache growing.
+  bool                                 mCacheSizeOnHardLimit;
   bool                                 mRemovingTrashDirs;
   nsCOMPtr<nsITimer>                   mTrashTimer;
   nsCOMPtr<nsIFile>                    mTrashDir;
   nsCOMPtr<nsIDirectoryEnumerator>     mTrashDirEnumerator;
   nsTArray<nsCString>                  mFailedTrashDirs;
   RefPtr<CacheFileContextEvictor>      mContextEvictor;
   TimeStamp                            mLastSmartSizeTime;
 };