Bug 681546 - Avoid large cache evictions as disk fills by smoothing "smart" max cache size.
authorMichal Novotny <michal.novotny@gmail.com>
Wed, 19 Oct 2011 14:35:57 +0200
changeset 78940 604f0f9eabdaa34054221b50edc190fec9a90050
parent 78939 e1c79e98e4ee7a02762d04ca13a71f05864a77a6
child 78941 8a237243d5ca53c4bf6c1558c01b58f3b916d56f
push id21351
push usermak77@bonardo.net
push dateThu, 20 Oct 2011 09:35:01 +0000
treeherdermozilla-central@67673422f7d2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs681546
milestone10.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 681546 - Avoid large cache evictions as disk fills by smoothing "smart" max cache size.
netwerk/cache/nsCacheService.cpp
netwerk/cache/nsCacheService.h
--- a/netwerk/cache/nsCacheService.cpp
+++ b/netwerk/cache/nsCacheService.cpp
@@ -166,17 +166,18 @@ public:
     bool            OfflineCacheEnabled();
     PRInt32         OfflineCacheCapacity()         { return mOfflineCacheCapacity; }
     nsILocalFile *  OfflineCacheParentDirectory()  { return mOfflineCacheParentDirectory; }
     
     bool            MemoryCacheEnabled();
     PRInt32         MemoryCacheCapacity();
     PRInt32         MemoryCacheMaxEntrySize()     { return mMemoryCacheMaxEntrySize; }
 
-    static PRUint32 GetSmartCacheSize(const nsAString& cachePath);
+    static PRUint32 GetSmartCacheSize(const nsAString& cachePath,
+                                      PRUint32 currentSize);
 
 private:
     bool                    PermittedToSmartSize(nsIPrefBranch*, bool firstRun);
     bool                    mHaveProfile;
     
     bool                    mDiskCacheEnabled;
     PRInt32                 mDiskCacheCapacity; // in kilobytes
     PRInt32                 mDiskCacheMaxEntrySize; // in kilobytes
@@ -195,18 +196,18 @@ private:
 
 NS_IMPL_THREADSAFE_ISUPPORTS1(nsCacheProfilePrefObserver, nsIObserver)
 
 // Runnable sent to main thread after the cache IO thread calculates available
 // disk space, so that there is no race in setting mDiskCacheCapacity.
 class nsSetSmartSizeEvent: public nsRunnable 
 {
 public:
-    nsSetSmartSizeEvent(bool firstRun, PRInt32 smartSize) 
-        : mFirstRun(firstRun) , mSmartSize(smartSize) {}
+    nsSetSmartSizeEvent(PRInt32 smartSize)
+        : mSmartSize(smartSize) {}
 
     NS_IMETHOD Run() 
     {
         nsresult rv;
         NS_ASSERTION(NS_IsMainThread(), 
                      "Setting smart size data off the main thread");
 
         // Main thread may have already called nsCacheService::Shutdown
@@ -221,57 +222,51 @@ public:
         }
         // ensure smart sizing wasn't switched off while event was pending
         rv = branch->GetBoolPref(DISK_CACHE_SMART_SIZE_ENABLED_PREF,
                                  &smartSizeEnabled);
         if (NS_FAILED(rv)) 
             smartSizeEnabled = false;
         if (smartSizeEnabled) {
             nsCacheService::SetDiskCacheCapacity(mSmartSize);
-            // also set on observer, in case mDiskDevice not init'd yet.
-            nsCacheService::gService->mObserver->SetDiskCacheCapacity(mSmartSize);
             rv = branch->SetIntPref(DISK_CACHE_SMART_SIZE_PREF, mSmartSize);
             if (NS_FAILED(rv)) 
                 NS_WARNING("Failed to set smart size pref");
         }
         return rv;
     }
 
-private: 
-    bool mFirstRun;
+private:
     PRInt32 mSmartSize;
 };
 
 
 // Runnable sent from main thread to cacheIO thread
 class nsGetSmartSizeEvent: public nsRunnable
 {
 public:
-    nsGetSmartSizeEvent(bool firstRun, const nsAString& cachePath) 
-      : mFirstRun(firstRun)
-      , mCachePath(cachePath)
-      , mSmartSize(0) 
+    nsGetSmartSizeEvent(const nsAString& cachePath, PRUint32 currentSize)
+      : mCachePath(cachePath)
+      , mCurrentSize(currentSize)
     {}
    
     // Calculates user's disk space available on a background thread and
     // dispatches this value back to the main thread.
     NS_IMETHOD Run()
     {
-        mSmartSize = 
-          nsCacheProfilePrefObserver::GetSmartCacheSize(mCachePath);
-        nsCOMPtr<nsIRunnable> event = new nsSetSmartSizeEvent(mFirstRun,
-                                                              mSmartSize);
-        NS_DispatchToMainThread(event);
+        PRUint32 size;
+        size = nsCacheProfilePrefObserver::GetSmartCacheSize(mCachePath,
+                                                             mCurrentSize);
+        NS_DispatchToMainThread(new nsSetSmartSizeEvent(size));
         return NS_OK;
     }
 
-private: 
-    bool mFirstRun;
+private:
     nsString mCachePath;
-    PRInt32 mSmartSize;
+    PRUint32 mCurrentSize;
 };
 
 class nsBlockOnCacheThreadEvent : public nsRunnable {
 public:
     nsBlockOnCacheThreadEvent()
     {
     }
     NS_IMETHOD Run()
@@ -430,28 +425,17 @@ nsCacheProfilePrefObserver::Observe(nsIS
             // Is the update because smartsizing was turned on, or off?
             bool smartSizeEnabled;
             rv = branch->GetBoolPref(DISK_CACHE_SMART_SIZE_ENABLED_PREF,
                                      &smartSizeEnabled);
             if (NS_FAILED(rv)) 
                 return rv;
             PRInt32 newCapacity = 0;
             if (smartSizeEnabled) {
-                // Dispatch event to update smart size: just keep using old
-                // value if this fails at any point
-                if (!mDiskCacheParentDirectory) 
-                    return NS_ERROR_NOT_AVAILABLE; // disk cache disabled anyway
-                nsAutoString cachePath;
-                rv = mDiskCacheParentDirectory->GetPath(cachePath);
-                if (NS_FAILED(rv)) 
-                    return rv;
-                // Smart sizing switched on: recalculate the capacity.
-                nsCOMPtr<nsIRunnable> event = 
-                    new nsGetSmartSizeEvent(false, cachePath);
-                rv = nsCacheService::DispatchToCacheIOThread(event);
+                nsCacheService::SetDiskSmartSize();
             } else {
                 // Smart sizing switched off: use user specified size
                 rv = branch->GetIntPref(DISK_CACHE_CAPACITY_PREF, &newCapacity);
                 if (NS_FAILED(rv)) 
                     return rv;
                 mDiskCacheCapacity = NS_MAX(0, newCapacity);
                 nsCacheService::SetDiskCacheCapacity(mDiskCacheCapacity);
             }
@@ -554,67 +538,82 @@ nsCacheProfilePrefObserver::Observe(nsIS
                                        &mOfflineCacheEnabled);
             nsCacheService::SetOfflineCacheEnabled(OfflineCacheEnabled());
         }
     }
     
     return NS_OK;
 }
 
+// Returns default ("smart") size (in KB) of cache, given available disk space
+// (also in KB)
+static PRUint32
+SmartCacheSize(const PRUint32 availKB)
+{
+    if (availKB > 100 * 1024 * 1024)
+        return MAX_CACHE_SIZE;  // skip computing if we're over 100 GB
+
+    // Grow/shrink in 10 MB units, deliberately, so that in the common case we
+    // don't shrink cache and evict items every time we startup (it's important
+    // that we don't slow down startup benchmarks).
+    PRUint32 sz10MBs = 0;
+    PRUint32 avail10MBs = availKB / (1024*10);
+
+    // .5% of space above 25 GB
+    if (avail10MBs > 2500) {
+        sz10MBs += (avail10MBs - 2500)*.005;
+        avail10MBs = 2500;
+    }
+    // 1% of space between 7GB -> 25 GB
+    if (avail10MBs > 700) {
+        sz10MBs += (avail10MBs - 700)*.01;
+        avail10MBs = 700;
+    }
+    // 5% of space between 500 MB -> 7 GB
+    if (avail10MBs > 50) {
+        sz10MBs += (avail10MBs - 50)*.05;
+        avail10MBs = 50;
+    }
+
+    // 40% of space up to 500 MB (50 MB min)
+    sz10MBs += NS_MAX<PRUint32>(5, avail10MBs * .4);
+
+    return NS_MIN<PRUint32>(MAX_CACHE_SIZE, sz10MBs * 10 * 1024);
+}
+
  /* Computes our best guess for the default size of the user's disk cache, 
   * based on the amount of space they have free on their hard drive. 
   * We use a tiered scheme: the more space available, 
   * the larger the disk cache will be. However, we do not want
   * to enable the disk cache to grow to an unbounded size, so the larger the
   * user's available space is, the smaller of a percentage we take. We set a
   * lower bound of 50MB and an upper bound of 1GB.  
   *
   *@param:  None.
   *@return: The size that the user's disk cache should default to, in kBytes.
   */
 PRUint32
-nsCacheProfilePrefObserver::GetSmartCacheSize(const nsAString& cachePath) 
+nsCacheProfilePrefObserver::GetSmartCacheSize(const nsAString& cachePath,
+                                              PRUint32 currentSize)
 {
-  // Check for free space on device where cache directory lives
-  nsresult rv;
-  nsCOMPtr<nsILocalFile> 
-      cacheDirectory (do_CreateInstance(NS_LOCAL_FILE_CONTRACTID, &rv));
-  if (NS_FAILED(rv) || !cacheDirectory)
-    return DEFAULT_CACHE_SIZE;
-  rv = cacheDirectory->InitWithPath(cachePath);
-  if (NS_FAILED(rv))
-    return DEFAULT_CACHE_SIZE;
-  PRInt64 bytesAvailable;
-  rv = cacheDirectory->GetDiskSpaceAvailable(&bytesAvailable);
-  if (NS_FAILED(rv))
-    return DEFAULT_CACHE_SIZE;
-  PRInt64 kBytesAvail = bytesAvailable / 1024;
-  
-  // 0 MB <= Available < 500 MB: Use between 50MB and 200MB
-  if (kBytesAvail < DEFAULT_CACHE_SIZE * 2) 
-    return NS_MAX<PRInt64>(MIN_CACHE_SIZE, kBytesAvail * 4 / 10);
-  
-  // 500MB <= Available < 2.5 GB: Use 250MB 
-  if (kBytesAvail < static_cast<PRInt64>(DEFAULT_CACHE_SIZE) * 10) 
-    return DEFAULT_CACHE_SIZE;
-
-  // 2.5 GB <= Available < 5 GB: Use between 250MB and 500MB
-  if (kBytesAvail < static_cast<PRInt64>(DEFAULT_CACHE_SIZE) * 20) 
-    return kBytesAvail / 10;
-
-  // 5 GB <= Available < 50 GB:  Use 625MB
-  if (kBytesAvail < static_cast<PRInt64>(DEFAULT_CACHE_SIZE) * 200 ) 
-    return DEFAULT_CACHE_SIZE * 5 / 2;
-
-  // 50 GB <= Available < 75 GB: Use 800MB
-  if (kBytesAvail < static_cast<PRInt64>(DEFAULT_CACHE_SIZE) * 300) 
-    return DEFAULT_CACHE_SIZE / 5 * 16;  
-  
-  // Use 1 GB
-  return MAX_CACHE_SIZE;
+    // Check for free space on device where cache directory lives
+    nsresult rv;
+    nsCOMPtr<nsILocalFile> 
+        cacheDirectory (do_CreateInstance(NS_LOCAL_FILE_CONTRACTID, &rv));
+    if (NS_FAILED(rv) || !cacheDirectory)
+        return DEFAULT_CACHE_SIZE;
+    rv = cacheDirectory->InitWithPath(cachePath);
+    if (NS_FAILED(rv))
+        return DEFAULT_CACHE_SIZE;
+    PRInt64 bytesAvailable;
+    rv = cacheDirectory->GetDiskSpaceAvailable(&bytesAvailable);
+    if (NS_FAILED(rv))
+        return DEFAULT_CACHE_SIZE;
+
+    return SmartCacheSize((bytesAvailable / 1024) + currentSize);
 }
 
 /* Determine if we are permitted to dynamically size the user's disk cache based
  * on their disk space available. We may do this so long as the pref 
  * smart_size.enabled is true.
  */
 bool
 nsCacheProfilePrefObserver::PermittedToSmartSize(nsIPrefBranch* branch, bool
@@ -728,23 +727,16 @@ nsCacheProfilePrefObserver::ReadPrefs(ns
                 PRInt32 oldCapacity;
                 rv = branch->GetIntPref(DISK_CACHE_CAPACITY_PREF, &oldCapacity);
                 if (NS_SUCCEEDED(rv)) {
                     mDiskCacheCapacity = oldCapacity;
                 } else {
                     mDiskCacheCapacity = DEFAULT_CACHE_SIZE;
                 }
             }
-            nsAutoString cachePath;
-            rv = mDiskCacheParentDirectory->GetPath(cachePath);
-            if (NS_SUCCEEDED(rv)) {
-                nsCOMPtr<nsIRunnable> event = 
-                    new nsGetSmartSizeEvent(!!firstSmartSizeRun, cachePath);
-                nsCacheService::DispatchToCacheIOThread(event);
-            }
         }
 
         if (firstSmartSizeRun) {
             // It is no longer our first run
             rv = branch->SetBoolPref(DISK_CACHE_SMART_SIZE_FIRST_RUN_PREF, 
                                      false);
             if (NS_FAILED(rv)) 
                 NS_WARNING("Failed setting first_run pref in ReadPrefs.");
@@ -1364,16 +1356,19 @@ nsCacheService::CreateDiskDevice()
         printf("### mDiskDevice->Init() failed (0x%.8x)\n", rv);
         printf("###    - disabling disk cache for this session.\n");
         printf("###\n");
 #endif        
         mEnableDiskDevice = false;
         delete mDiskDevice;
         mDiskDevice = nsnull;
     }
+
+    SetDiskSmartSize_Locked(true);
+
     return rv;
 }
 
 nsresult
 nsCacheService::CreateOfflineDevice()
 {
     CACHE_LOG_ALWAYS(("Creating offline device"));
 
@@ -2550,8 +2545,54 @@ nsCacheService::OnEnterExitPrivateBrowsi
 
     gService->DoomActiveEntries();
 
     if (gService->mMemoryDevice) {
         // clear memory cache
         gService->mMemoryDevice->EvictEntries(nsnull);
     }
 }
+
+nsresult
+nsCacheService::SetDiskSmartSize()
+{
+    nsCacheServiceAutoLock lock;
+
+    if (!gService) return NS_ERROR_NOT_AVAILABLE;
+
+    return gService->SetDiskSmartSize_Locked(false);
+}
+
+nsresult
+nsCacheService::SetDiskSmartSize_Locked(bool checkPref)
+{
+    nsresult rv;
+
+    if (!mObserver->DiskCacheParentDirectory())
+        return NS_ERROR_NOT_AVAILABLE;
+
+    if (!mDiskDevice)
+        return NS_ERROR_NOT_AVAILABLE;
+
+    if (checkPref) {
+        nsCOMPtr<nsIPrefBranch2> branch = do_GetService(NS_PREFSERVICE_CONTRACTID);
+        if (!branch) return NS_ERROR_FAILURE;
+
+        bool smartSizeEnabled;
+        rv = branch->GetBoolPref(DISK_CACHE_SMART_SIZE_ENABLED_PREF,
+                                 &smartSizeEnabled);
+
+        if (NS_FAILED(rv) || !smartSizeEnabled)
+            return NS_ERROR_NOT_AVAILABLE;
+    }
+
+    nsAutoString cachePath;
+    rv = mObserver->DiskCacheParentDirectory()->GetPath(cachePath);
+    if (NS_SUCCEEDED(rv)) {
+        nsCOMPtr<nsIRunnable> event =
+            new nsGetSmartSizeEvent(cachePath, mDiskDevice->getCacheSize());
+        DispatchToCacheIOThread(event);
+    } else {
+        return NS_ERROR_FAILURE;
+    }
+
+    return NS_OK;
+}
--- a/netwerk/cache/nsCacheService.h
+++ b/netwerk/cache/nsCacheService.h
@@ -173,16 +173,19 @@ public:
     static void      SetOfflineCacheEnabled(bool    enabled);
     // Sets the offline cache capacity (in kilobytes)
     static void      SetOfflineCacheCapacity(PRInt32  capacity);
 
     static void      SetMemoryCache();
 
     static void      OnEnterExitPrivateBrowsing();
 
+    // Starts smart cache size computation if disk device is available
+    static nsresult  SetDiskSmartSize();
+
     nsresult         Init();
     void             Shutdown();
 
     static void      AssertOwnsLock()
     { gService->mLock.AssertCurrentThreadOwns(); }
 
 private:
     friend class nsCacheServiceAutoLock;
@@ -253,16 +256,18 @@ private:
     PLDHashOperator  RemoveActiveEntry(PLDHashTable *    table,
                                        PLDHashEntryHdr * hdr,
                                        PRUint32          number,
                                        void *            arg);
 #if defined(PR_LOGGING)
     void LogCacheStatistics();
 #endif
 
+    nsresult         SetDiskSmartSize_Locked(bool checkPref);
+
     /**
      *  Data Members
      */
 
     static nsCacheService *         gService;  // there can be only one...
     
     nsCacheProfilePrefObserver *    mObserver;