Bug 913822 - Shutdown I/O time limit for HTTP cache. r=michal, a=sylvestre
authorHonza Bambas <honzab.moz@firemni.cz>
Wed, 13 Jan 2016 16:11:00 -0500
changeset 312938 e4b4c84f6607e81d69bb4895edd67153c47bd7ed
parent 312937 060086e4af2ea06b0d6047004d2fc6ff543a1452
child 312939 795d3cdbcca45052cee7172df319c734a8930221
push id1040
push userraliiev@mozilla.com
push dateMon, 29 Feb 2016 17:11:22 +0000
treeherdermozilla-release@8c3167321162 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmichal, sylvestre
bugs913822
milestone45.0
Bug 913822 - Shutdown I/O time limit for HTTP cache. r=michal, a=sylvestre
modules/libpref/init/all.js
netwerk/cache2/CacheFileIOManager.cpp
netwerk/cache2/CacheFileIOManager.h
netwerk/cache2/CacheObserver.cpp
netwerk/cache2/CacheObserver.h
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -84,16 +84,21 @@ pref("browser.cache.disk_cache_ssl",    
 pref("browser.cache.check_doc_frequency",   3);
 // Limit of recent metadata we keep in memory for faster access, in Kb
 pref("browser.cache.disk.metadata_memory_limit", 250); // 0.25 MB
 // The number of chunks we preload ahead of read.  One chunk has currently 256kB.
 pref("browser.cache.disk.preload_chunk_count", 4); // 1 MB of read ahead
 // The half life used to re-compute cache entries frecency in hours.
 pref("browser.cache.frecency_half_life_hours", 6);
 
+// Number of seconds the cache spends writting pending data and closing files
+// after the shutdown has been signalled.  Past that time data are never written
+// and files are left open given up to the OS to do the cleanup.
+pref("browser.cache.max_shutdown_io_lag", 2);
+
 pref("browser.cache.offline.enable",           true);
 // enable offline apps by default, disable prompt
 pref("offline-apps.allow_by_default",          true);
 
 // offline cache capacity in kilobytes
 pref("browser.cache.offline.capacity",         512000);
 
 // the user should be warned if offline app disk usage exceeds this amount
--- a/netwerk/cache2/CacheFileIOManager.cpp
+++ b/netwerk/cache2/CacheFileIOManager.cpp
@@ -531,40 +531,57 @@ CacheFileHandles::SizeOfExcludingThis(mo
 
 // Events
 
 class ShutdownEvent : public nsRunnable {
 public:
   ShutdownEvent(mozilla::Mutex *aLock, mozilla::CondVar *aCondVar)
     : mLock(aLock)
     , mCondVar(aCondVar)
+    , mPrepare(true)
   {
     MOZ_COUNT_CTOR(ShutdownEvent);
   }
 
 protected:
   ~ShutdownEvent()
   {
     MOZ_COUNT_DTOR(ShutdownEvent);
   }
 
 public:
   NS_IMETHOD Run()
   {
+    if (mPrepare) {
+      MOZ_ASSERT(CacheFileIOManager::gInstance->mIOThread->IsCurrentThread());
+
+      mPrepare = false;
+
+      // This event is first posted to the XPCOM level (executed ASAP) of the IO thread
+      // and sets the timestamp of the shutdown start.  This will cause some operations
+      // to be bypassed when due (actually leak most of the open files).
+      CacheFileIOManager::gInstance->mShutdownDemandedTime = TimeStamp::NowLoRes();
+
+      // Redispatch to the right level to proceed with shutdown.
+      CacheFileIOManager::gInstance->mIOThread->Dispatch(this, CacheIOThread::CLOSE);
+      return NS_OK;
+    }
+
     MutexAutoLock lock(*mLock);
 
     CacheFileIOManager::gInstance->ShutdownInternal();
 
     mCondVar->Notify();
     return NS_OK;
   }
 
 protected:
   mozilla::Mutex   *mLock;
   mozilla::CondVar *mCondVar;
+  bool              mPrepare;
 };
 
 class OpenFileEvent : public nsRunnable {
 public:
   OpenFileEvent(const nsACString &aKey, uint32_t aFlags,
                 CacheFileIOListener *aCallback)
     : mFlags(aFlags)
     , mCallback(aCallback)
@@ -692,17 +709,23 @@ protected:
   }
 
 public:
   NS_IMETHOD Run()
   {
     nsresult rv;
 
     if (mHandle->IsClosed()) {
-      rv = NS_ERROR_NOT_INITIALIZED;
+      // We usually get here only after the internal shutdown
+      // (i.e. mShuttingDown == true).  Pretend write has succeeded
+      // to avoid any past-shutdown file dooming.
+      rv = (CacheFileIOManager::gInstance->IsPastShutdownIOLag() ||
+            CacheFileIOManager::gInstance->mShuttingDown)
+        ? NS_OK
+        : NS_ERROR_NOT_INITIALIZED;
     } else {
       rv = CacheFileIOManager::gInstance->WriteInternal(
           mHandle, mOffset, mBuf, mCount, mValidate, mTruncate);
       if (NS_FAILED(rv) && !mCallback) {
         // No listener is going to handle the error, doom the file
         CacheFileIOManager::gInstance->DoomFileInternal(mHandle);
       }
     }
@@ -1143,17 +1166,19 @@ CacheFileIOManager::Shutdown()
 
   {
     mozilla::Mutex lock("CacheFileIOManager::Shutdown() lock");
     mozilla::CondVar condVar(lock, "CacheFileIOManager::Shutdown() condVar");
 
     MutexAutoLock autoLock(lock);
     RefPtr<ShutdownEvent> ev = new ShutdownEvent(&lock, &condVar);
     DebugOnly<nsresult> rv;
-    rv = gInstance->mIOThread->Dispatch(ev, CacheIOThread::CLOSE);
+    nsCOMPtr<nsIEventTarget> ioTarget = gInstance->mIOThread->Target();
+    MOZ_ASSERT(ioTarget);
+    rv = ioTarget->Dispatch(ev, nsIEventTarget::DISPATCH_NORMAL);
     MOZ_ASSERT(NS_SUCCEEDED(rv));
     condVar.Wait();
   }
 
   MOZ_ASSERT(gInstance->mHandles.HandleCount() == 0);
   MOZ_ASSERT(gInstance->mHandlesByLastUsed.Length() == 0);
 
   if (gInstance->mIOThread) {
@@ -1235,16 +1260,36 @@ CacheFileIOManager::ShutdownInternal()
   if (mTrashDirEnumerator) {
     mTrashDirEnumerator->Close();
     mTrashDirEnumerator = nullptr;
   }
 
   return NS_OK;
 }
 
+bool
+CacheFileIOManager::IsPastShutdownIOLag()
+{
+#ifdef DEBUG
+  return false;
+#endif
+
+  if (mShutdownDemandedTime.IsNull()) {
+    return false;
+  }
+
+  TimeDuration const& preferredIOLag = CacheObserver::MaxShutdownIOLag();
+  if (preferredIOLag < TimeDuration(0)) {
+    return false;
+  }
+
+  TimeDuration currentIOLag = TimeStamp::NowLoRes() - mShutdownDemandedTime;
+  return currentIOLag > preferredIOLag;
+}
+
 // static
 nsresult
 CacheFileIOManager::OnProfile()
 {
   LOG(("CacheFileIOManager::OnProfile() [gInstance=%p]", gInstance));
 
   RefPtr<CacheFileIOManager> ioMan = gInstance;
   if (!ioMan) {
@@ -1920,16 +1965,23 @@ CacheFileIOManager::WriteInternal(CacheF
                                   bool aValidate, bool aTruncate)
 {
   LOG(("CacheFileIOManager::WriteInternal() [handle=%p, offset=%lld, count=%d, "
        "validate=%d, truncate=%d]", aHandle, aOffset, aCount, aValidate,
        aTruncate));
 
   nsresult rv;
 
+  if (IsPastShutdownIOLag()) {
+    LOG(("  past the shutdown I/O lag, nothing written"));
+    // Pretend the write has succeeded, otherwise upper layers will doom
+    // the file and we end up with I/O anyway.
+    return NS_OK;
+  }
+
   if (!aHandle->mFileExists) {
     rv = CreateFile(aHandle);
     NS_ENSURE_SUCCESS(rv, rv);
   }
 
   if (!aHandle->mFD) {
     rv = OpenNSPRHandle(aHandle);
     NS_ENSURE_SUCCESS(rv, rv);
@@ -2077,17 +2129,17 @@ CacheFileIOManager::DoomFileInternal(Cac
       LOG(("  pinning status not known, deferring doom decision"));
       return NS_OK;
     }
   }
 
   if (aHandle->mFileExists) {
     // we need to move the current file to the doomed directory
     if (aHandle->mFD) {
-      ReleaseNSPRHandleInternal(aHandle);
+      ReleaseNSPRHandleInternal(aHandle, true);
     }
 
     // find unused filename
     nsCOMPtr<nsIFile> file;
     rv = GetDoomedFile(getter_AddRefs(file));
     NS_ENSURE_SUCCESS(rv, rv);
 
     nsCOMPtr<nsIFile> parentDir;
@@ -2224,28 +2276,39 @@ CacheFileIOManager::ReleaseNSPRHandle(Ca
   RefPtr<ReleaseNSPRHandleEvent> ev = new ReleaseNSPRHandleEvent(aHandle);
   rv = ioMan->mIOThread->Dispatch(ev, CacheIOThread::CLOSE);
   NS_ENSURE_SUCCESS(rv, rv);
 
   return NS_OK;
 }
 
 nsresult
-CacheFileIOManager::ReleaseNSPRHandleInternal(CacheFileHandle *aHandle)
+CacheFileIOManager::ReleaseNSPRHandleInternal(CacheFileHandle *aHandle,
+                                              bool aIgnoreShutdownLag)
 {
   LOG(("CacheFileIOManager::ReleaseNSPRHandleInternal() [handle=%p]", aHandle));
 
   MOZ_ASSERT(CacheFileIOManager::IsOnIOThreadOrCeased());
   MOZ_ASSERT(aHandle->mFD);
 
   DebugOnly<bool> found;
   found = mHandlesByLastUsed.RemoveElement(aHandle);
   MOZ_ASSERT(found);
 
-  PR_Close(aHandle->mFD);
+  if (aIgnoreShutdownLag || !IsPastShutdownIOLag()) {
+    PR_Close(aHandle->mFD);
+  } else {
+    // Pretend this file has been validated (the metadata has been written)
+    // to prevent removal I/O on this apparently used file.  The entry will
+    // never be used, since it doesn't have correct metadata, thus we don't
+    // need to worry about removing it.
+    aHandle->mInvalid = false;
+    LOG(("  past the shutdown I/O lag, leaking file handle"));
+  }
+
   aHandle->mFD = nullptr;
 
   return NS_OK;
 }
 
 // static
 nsresult
 CacheFileIOManager::TruncateSeekSetEOF(CacheFileHandle *aHandle,
@@ -2533,17 +2596,17 @@ CacheFileIOManager::RenameFileInternal(C
   }
 
   if (!aHandle->FileExists()) {
     aHandle->mKey = aNewName;
     return NS_OK;
   }
 
   if (aHandle->mFD) {
-    ReleaseNSPRHandleInternal(aHandle);
+    ReleaseNSPRHandleInternal(aHandle, true);
   }
 
   rv = aHandle->mFile->MoveToNative(nullptr, aNewName);
   NS_ENSURE_SUCCESS(rv, rv);
 
   aHandle->mKey = aNewName;
   return NS_OK;
 }
@@ -3726,17 +3789,17 @@ CacheFileIOManager::OpenNSPRHandle(Cache
   MOZ_ASSERT(mHandlesByLastUsed.Length() <= kOpenHandlesLimit);
   MOZ_ASSERT((aCreate && !aHandle->mFileExists) ||
              (!aCreate && aHandle->mFileExists));
 
   nsresult rv;
 
   if (mHandlesByLastUsed.Length() == kOpenHandlesLimit) {
     // close handle that hasn't been used for the longest time
-    rv = ReleaseNSPRHandleInternal(mHandlesByLastUsed[0]);
+    rv = ReleaseNSPRHandleInternal(mHandlesByLastUsed[0], true);
     NS_ENSURE_SUCCESS(rv, rv);
   }
 
   if (aCreate) {
     rv = aHandle->mFile->OpenNSPRFileDesc(
            PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE, 0600, &aHandle->mFD);
     if (rv == NS_ERROR_FILE_ALREADY_EXISTS ||  // error from nsLocalFileWin
         rv == NS_ERROR_FILE_NO_DEVICE_SPACE) { // error from nsLocalFileUnix
--- a/netwerk/cache2/CacheFileIOManager.h
+++ b/netwerk/cache2/CacheFileIOManager.h
@@ -377,17 +377,18 @@ private:
   nsresult ReadInternal(CacheFileHandle *aHandle, int64_t aOffset,
                         char *aBuf, int32_t aCount);
   nsresult WriteInternal(CacheFileHandle *aHandle, int64_t aOffset,
                          const char *aBuf, int32_t aCount, bool aValidate,
                          bool aTruncate);
   nsresult DoomFileInternal(CacheFileHandle *aHandle,
                             PinningDoomRestriction aPinningStatusRestriction = NO_RESTRICTION);
   nsresult DoomFileByKeyInternal(const SHA1Sum::Hash *aHash);
-  nsresult ReleaseNSPRHandleInternal(CacheFileHandle *aHandle);
+  nsresult ReleaseNSPRHandleInternal(CacheFileHandle *aHandle,
+                                     bool aIgnoreShutdownLag = false);
   nsresult TruncateSeekSetEOFInternal(CacheFileHandle *aHandle,
                                       int64_t aTruncatePos, int64_t aEOFPos);
   nsresult RenameFileInternal(CacheFileHandle *aHandle,
                               const nsACString &aNewName);
   nsresult EvictIfOverLimitInternal();
   nsresult OverLimitEvictionInternal();
   nsresult EvictAllInternal();
   nsresult EvictByContextInternal(nsILoadContextInfo *aLoadContextInfo,
@@ -424,21 +425,28 @@ private:
   nsresult CacheIndexStateChangedInternal();
 
   // Smart size calculation. UpdateSmartCacheSize() must be called on IO thread.
   // It is called in EvictIfOverLimitInternal() just before we decide whether to
   // start overlimit eviction or not and also in OverLimitEvictionInternal()
   // before we start an eviction loop.
   nsresult UpdateSmartCacheSize(int64_t aFreeSpace);
 
+  // May return true after shutdown only when time for flushing all data
+  // has already passed.
+  bool IsPastShutdownIOLag();
+
   // Memory reporting (private part)
   size_t SizeOfExcludingThisInternal(mozilla::MallocSizeOf mallocSizeOf) const;
 
   static CacheFileIOManager           *gInstance;
   TimeStamp                            mStartTime;
+  // Shutdown time stamp, accessed only on the I/O thread.  Used to bypass
+  // I/O after a certain time pass the shutdown has been demanded.
+  TimeStamp                            mShutdownDemandedTime;
   bool                                 mShuttingDown;
   RefPtr<CacheIOThread>                mIOThread;
   nsCOMPtr<nsIFile>                    mCacheDirectory;
 #if defined(MOZ_WIDGET_ANDROID)
   // On Android we add the active profile directory name between the path
   // and the 'cache2' leaf name.  However, to delete any leftover data from
   // times before we were doing it, we still need to access the directory
   // w/o the profile name in the path.  Here it is stored.
--- a/netwerk/cache2/CacheObserver.cpp
+++ b/netwerk/cache2/CacheObserver.cpp
@@ -6,16 +6,17 @@
 
 #include "CacheStorageService.h"
 #include "CacheFileIOManager.h"
 #include "LoadContextInfo.h"
 #include "nsICacheStorage.h"
 #include "nsIObserverService.h"
 #include "mozilla/Services.h"
 #include "mozilla/Preferences.h"
+#include "mozilla/TimeStamp.h"
 #include "nsServiceManagerUtils.h"
 #include "prsystem.h"
 #include <time.h>
 #include <math.h>
 
 namespace mozilla {
 namespace net {
 
@@ -86,16 +87,19 @@ static bool kDefaultClearCacheOnShutdown
 bool CacheObserver::sClearCacheOnShutdown = kDefaultClearCacheOnShutdown;
 
 static bool kDefaultCacheFSReported = false;
 bool CacheObserver::sCacheFSReported = kDefaultCacheFSReported;
 
 static bool kDefaultHashStatsReported = false;
 bool CacheObserver::sHashStatsReported = kDefaultHashStatsReported;
 
+static int32_t const kDefaultMaxShutdownIOLag = 2; // seconds
+int32_t CacheObserver::sMaxShutdownIOLag = kDefaultMaxShutdownIOLag;
+
 NS_IMPL_ISUPPORTS(CacheObserver,
                   nsIObserver,
                   nsISupportsWeakReference)
 
 // static
 nsresult
 CacheObserver::Init()
 {
@@ -232,16 +236,19 @@ CacheObserver::AttachToPreferences()
       "browser.cache.frecency_half_life_hours", kDefaultHalfLifeHours)));
     break;
   }
 
   mozilla::Preferences::AddBoolVarCache(
     &sSanitizeOnShutdown, "privacy.sanitize.sanitizeOnShutdown", kDefaultSanitizeOnShutdown);
   mozilla::Preferences::AddBoolVarCache(
     &sClearCacheOnShutdown, "privacy.clearOnShutdown.cache", kDefaultClearCacheOnShutdown);
+
+  mozilla::Preferences::AddIntVarCache(
+    &sMaxShutdownIOLag, "browser.cache.max_shutdown_io_lag", kDefaultMaxShutdownIOLag);
 }
 
 // static
 uint32_t const CacheObserver::MemoryCacheCapacity()
 {
   if (sMemoryCacheCapacity >= 0)
     return sMemoryCacheCapacity << 10;
 
@@ -461,16 +468,23 @@ bool const CacheObserver::EntryIsTooBig(
     : (static_cast<int64_t>(MemoryCacheCapacity() >> 3));
 
   if (aSize > derivedLimit)
     return true;
 
   return false;
 }
 
+// static
+TimeDuration const& CacheObserver::MaxShutdownIOLag()
+{
+  static TimeDuration period = TimeDuration::FromSeconds(sMaxShutdownIOLag);
+  return period;
+}
+
 NS_IMETHODIMP
 CacheObserver::Observe(nsISupports* aSubject,
                        const char* aTopic,
                        const char16_t* aData)
 {
   if (!strcmp(aTopic, "prefservice:after-app-defaults")) {
     CacheFileIOManager::Init();
     return NS_OK;
--- a/netwerk/cache2/CacheObserver.h
+++ b/netwerk/cache2/CacheObserver.h
@@ -66,16 +66,18 @@ class CacheObserver : public nsIObserver
   static void SetCacheFSReported();
   static bool const HashStatsReported()
     { return sHashStatsReported; }
   static void SetHashStatsReported();
   static void ParentDirOverride(nsIFile ** aDir);
 
   static bool const EntryIsTooBig(int64_t aSize, bool aUsingDisk);
 
+  static TimeDuration const& MaxShutdownIOLag();
+
 private:
   static CacheObserver* sSelf;
 
   void StoreDiskCacheCapacity();
   void StoreCacheFSReported();
   void StoreHashStatsReported();
   void AttachToPreferences();
 
@@ -96,16 +98,17 @@ private:
   static uint32_t sMaxDiskPriorityChunksMemoryUsage;
   static uint32_t sCompressionLevel;
   static float sHalfLifeHours;
   static int32_t sHalfLifeExperiment;
   static bool sSanitizeOnShutdown;
   static bool sClearCacheOnShutdown;
   static bool sCacheFSReported;
   static bool sHashStatsReported;
+  static int32_t sMaxShutdownIOLag;
 
   // Non static properties, accessible via sSelf
   nsCOMPtr<nsIFile> mCacheParentDirectoryOverride;
 };
 
 } // namespace net
 } // namespace mozilla