Bug 699951 - add a memory reporter for heap usage by the disk cache. r=michal
authorNick Hurley <hurley@todesschaf.org>
Mon, 05 Nov 2012 10:22:33 -0800
changeset 112336 c87fcf6ed0a77d0f532b41217b584a806879c5b9
parent 112335 c999fc311c2859a4f35475a7745317bead62a2e7
child 112337 0dea4f9a91330ce98542cb2d131e385e39d3c878
push id1
push userroot
push dateMon, 20 Oct 2014 17:29:22 +0000
reviewersmichal
bugs699951
milestone19.0a1
Bug 699951 - add a memory reporter for heap usage by the disk cache. r=michal
netwerk/cache/nsCacheService.cpp
netwerk/cache/nsCacheService.h
netwerk/cache/nsDiskCacheBinding.cpp
netwerk/cache/nsDiskCacheBinding.h
netwerk/cache/nsDiskCacheBlockFile.cpp
netwerk/cache/nsDiskCacheBlockFile.h
netwerk/cache/nsDiskCacheDevice.cpp
netwerk/cache/nsDiskCacheDevice.h
netwerk/cache/nsDiskCacheMap.cpp
netwerk/cache/nsDiskCacheMap.h
netwerk/cache/nsDiskCacheStreams.cpp
netwerk/cache/nsDiskCacheStreams.h
toolkit/components/telemetry/Histograms.json
--- a/netwerk/cache/nsCacheService.cpp
+++ b/netwerk/cache/nsCacheService.cpp
@@ -1055,22 +1055,33 @@ private:
 /******************************************************************************
  * nsCacheService
  *****************************************************************************/
 nsCacheService *   nsCacheService::gService = nullptr;
 
 static nsCOMPtr<nsIMemoryReporter> MemoryCacheReporter = nullptr;
 
 NS_THREADSAFE_MEMORY_REPORTER_IMPLEMENT(NetworkMemoryCache,
-    "explicit/network-memory-cache",
+    "explicit/network/memory-cache",
     KIND_HEAP,
     UNITS_BYTES,
     nsCacheService::MemoryDeviceSize,
     "Memory used by the network memory cache.")
 
+NS_MEMORY_REPORTER_MALLOC_SIZEOF_FUN(NetworkDiskCacheSizeOfFun, "network-disk-cache")
+
+static nsCOMPtr<nsIMemoryReporter> DiskCacheReporter = nullptr;
+
+NS_THREADSAFE_MEMORY_REPORTER_IMPLEMENT(NetworkDiskCache,
+    "explicit/network/disk-cache",
+    KIND_HEAP,
+    UNITS_BYTES,
+    nsCacheService::DiskDeviceHeapSize,
+    "Memory used by the network disk cache.")
+
 NS_IMPL_THREADSAFE_ISUPPORTS1(nsCacheService, nsICacheService)
 
 nsCacheService::nsCacheService()
     : mLock("nsCacheService.mLock"),
       mCondVar(mLock, "nsCacheService.mCondVar"),
       mInitialized(false),
       mClearingEntries(false),
       mEnableMemoryDevice(true),
@@ -1201,21 +1212,24 @@ nsCacheService::Shutdown()
         (void) SyncWithCacheIOThread();
 
         // obtain the disk cache directory in case we need to sanitize it
         parentDir = mObserver->DiskCacheParentDirectory();
         shouldSanitize = mObserver->SanitizeAtShutdown();
         mObserver->Remove();
         NS_RELEASE(mObserver);
         
-        // unregister memory reporter, before deleting the memory device, just
+        // unregister memory reporters, before deleting the devices, just
         // to be safe
         NS_UnregisterMemoryReporter(MemoryCacheReporter);
         MemoryCacheReporter = nullptr;
 
+        NS_UnregisterMemoryReporter(DiskCacheReporter);
+        DiskCacheReporter = nullptr;
+
         // deallocate memory and disk caches
         delete mMemoryDevice;
         mMemoryDevice = nullptr;
 
         delete mDiskDevice;
         mDiskDevice = nullptr;
 
         if (mOfflineDevice)
@@ -1559,16 +1573,19 @@ nsCacheService::CreateDiskDevice()
             mSmartSizeTimer = nullptr;
         }
     } else {
         NS_WARNING("Can't create smart size timer");
     }
     // Ignore state of the timer and return success since the purpose of the
     // method (create the disk-device) has been fulfilled
 
+    DiskCacheReporter = new NS_MEMORY_REPORTER_NAME(NetworkDiskCache);
+    NS_RegisterMemoryReporter(DiskCacheReporter);
+
     return NS_OK;
 }
 
 // Runnable sent from cache thread to main thread
 class nsDisableOldMaxSmartSizePrefEvent: public nsRunnable
 {
 public:
     nsDisableOldMaxSmartSizePrefEvent() {}
@@ -2231,16 +2248,24 @@ nsCacheService::EnsureEntryHasDevice(nsC
 
 int64_t
 nsCacheService::MemoryDeviceSize()
 {
     nsMemoryCacheDevice *memoryDevice = GlobalInstance()->mMemoryDevice;
     return memoryDevice ? memoryDevice->TotalSize() : 0;
 }
 
+int64_t
+nsCacheService::DiskDeviceHeapSize()
+{
+    nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_DISKDEVICEHEAPSIZE));
+    nsDiskCacheDevice *diskDevice = GlobalInstance()->mDiskDevice;
+    return (int64_t)(diskDevice ? diskDevice->SizeOfIncludingThis(NetworkDiskCacheSizeOfFun) : 0);
+}
+
 nsresult
 nsCacheService::DoomEntry(nsCacheEntry * entry)
 {
     return gService->DoomEntry_Internal(entry, true);
 }
 
 
 nsresult
--- a/netwerk/cache/nsCacheService.h
+++ b/netwerk/cache/nsCacheService.h
@@ -124,16 +124,18 @@ public:
     /**
      * Methods called by any cache classes
      */
 
     static
     nsCacheService * GlobalInstance()   { return gService; }
 
     static int64_t   MemoryDeviceSize();
+
+    static int64_t   DiskDeviceHeapSize();
     
     static nsresult  DoomEntry(nsCacheEntry * entry);
 
     static bool      IsStorageEnabledForPolicy_Locked(nsCacheStoragePolicy policy);
 
     /**
      * Called by disk cache to notify us to use the new max smart size
      */
--- a/netwerk/cache/nsDiskCacheBinding.cpp
+++ b/netwerk/cache/nsDiskCacheBinding.cpp
@@ -363,8 +363,60 @@ nsDiskCacheBindery::ActiveBindings()
     NS_ASSERTION(initialized, "nsDiskCacheBindery not initialized");
     if (!initialized) return false;
 
     bool    activeBinding = false;
     PL_DHashTableEnumerate(&table, ActiveBinding, &activeBinding);
 
     return activeBinding;
 }
+
+struct AccumulatorArg {
+    size_t mUsage;
+    nsMallocSizeOfFun mMallocSizeOf;
+};
+
+PLDHashOperator
+AccumulateHeapUsage(PLDHashTable *table, PLDHashEntryHdr *hdr, uint32_t number,
+                    void *arg)
+{
+    nsDiskCacheBinding *binding = ((HashTableEntry *)hdr)->mBinding;
+    NS_ASSERTION(binding, "### disk cache binding = nsnull!");
+
+    AccumulatorArg *acc = (AccumulatorArg *)arg;
+
+    nsDiskCacheBinding *head = binding;
+    do {
+        acc->mUsage += acc->mMallocSizeOf(binding);
+
+        if (binding->mStreamIO) {
+            acc->mUsage += binding->mStreamIO->SizeOfIncludingThis(acc->mMallocSizeOf);
+        }
+
+        /* No good way to get at mDeactivateEvent internals for proper size, so
+           we use this as an estimate. */
+        if (binding->mDeactivateEvent) {
+            acc->mUsage += acc->mMallocSizeOf(binding->mDeactivateEvent);
+        }
+
+        binding = (nsDiskCacheBinding *)PR_NEXT_LINK(binding);
+    } while (binding != head);
+
+    return PL_DHASH_NEXT;
+}
+
+/**
+ * SizeOfExcludingThis: return the amount of heap memory (bytes) being used by the bindery
+ */
+size_t
+nsDiskCacheBindery::SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf)
+{
+    NS_ASSERTION(initialized, "nsDiskCacheBindery not initialized");
+    if (!initialized) return 0;
+
+    AccumulatorArg arg;
+    arg.mUsage = 0;
+    arg.mMallocSizeOf = aMallocSizeOf;
+
+    PL_DHashTableEnumerate(&table, AccumulateHeapUsage, &arg);
+
+    return arg.mUsage;
+}
--- a/netwerk/cache/nsDiskCacheBinding.h
+++ b/netwerk/cache/nsDiskCacheBinding.h
@@ -99,17 +99,19 @@ public:
     void                    Reset();
 
     nsDiskCacheBinding *    CreateBinding(nsCacheEntry *       entry,
                                           nsDiskCacheRecord *  record);
 
     nsDiskCacheBinding *    FindActiveBinding(uint32_t  hashNumber);
     void                    RemoveBinding(nsDiskCacheBinding * binding);
     bool                    ActiveBindings();
-    
+
+    size_t                 SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf);
+
 private:
     nsresult                AddBinding(nsDiskCacheBinding * binding);
 
     // member variables
     static PLDHashTableOps ops;
     PLDHashTable           table;
     bool                   initialized;
 };
--- a/netwerk/cache/nsDiskCacheBlockFile.cpp
+++ b/netwerk/cache/nsDiskCacheBlockFile.cpp
@@ -388,8 +388,14 @@ nsDiskCacheBlockFile::Write(int32_t offs
         mFileSize = NS_MIN(mFileSize, maxFileSize);
         //  Appears to cause bug 617123?  Disabled for now.
         //mozilla::fallocate(mFD, mFileSize);
     }
     if (PR_Seek(mFD, offset, PR_SEEK_SET) != offset)
         return false;
     return PR_Write(mFD, buf, amount) == amount;
 }
+
+size_t
+nsDiskCacheBlockFile::SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf)
+{
+    return aMallocSizeOf(mBitMap) + aMallocSizeOf(mFD);
+}
--- a/netwerk/cache/nsDiskCacheBlockFile.h
+++ b/netwerk/cache/nsDiskCacheBlockFile.h
@@ -39,17 +39,19 @@ public:
      * Truncates the block file to the end of the last allocated block.
      */
     nsresult  Trim() { return nsDiskCache::Truncate(mFD, CalcBlockFileSize()); }
     nsresult  DeallocateBlocks( int32_t  startBlock, int32_t  numBlocks);
     nsresult  WriteBlocks( void * buffer, uint32_t size, int32_t  numBlocks, 
                            int32_t * startBlock);
     nsresult  ReadBlocks( void * buffer, int32_t  startBlock, int32_t  numBlocks, 
                           int32_t * bytesRead);
-    
+
+    size_t   SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf);
+
 private:
     nsresult  FlushBitMap();
     int32_t   AllocateBlocks( int32_t  numBlocks);
     nsresult  VerifyAllocation( int32_t startBlock, int32_t numBLocks);
     uint32_t  CalcBlockFileSize();
     bool   Write(int32_t offset, const void *buf, int32_t amount);
 
 /**
--- a/netwerk/cache/nsDiskCacheDevice.cpp
+++ b/netwerk/cache/nsDiskCacheDevice.cpp
@@ -1155,8 +1155,20 @@ nsDiskCacheDevice::SetMaxEntrySize(int32
 {
     // Internal units are bytes. Changing this only takes effect *after* the
     // change and has no consequences for existing cache-entries
     if (maxSizeInKilobytes >= 0)
         mMaxEntrySize = maxSizeInKilobytes * 1024;
     else
         mMaxEntrySize = -1;
 }
+
+size_t
+nsDiskCacheDevice::SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf)
+{
+    size_t usage = aMallocSizeOf(this);
+
+    usage += mCacheMap.SizeOfExcludingThis(aMallocSizeOf);
+    usage += mBindery.SizeOfExcludingThis(aMallocSizeOf);
+
+    return usage;
+}
+
--- a/netwerk/cache/nsDiskCacheDevice.h
+++ b/netwerk/cache/nsDiskCacheDevice.h
@@ -49,16 +49,18 @@ public:
     virtual nsresult        OnDataSizeChange(nsCacheEntry * entry, int32_t deltaSize);
     
     virtual nsresult        Visit(nsICacheVisitor * visitor);
 
     virtual nsresult        EvictEntries(const char * clientID);
 
     bool                    EntryIsTooBig(int64_t entrySize);
 
+    size_t                 SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf);
+
     /**
      * Preference accessors
      */
     void                    SetCacheParentDirectory(nsIFile * parentDir);
     void                    SetCapacity(uint32_t  capacity);
     void                    SetMaxEntrySize(int32_t  maxSizeInKilobytes);
 
 /* private: */
--- a/netwerk/cache/nsDiskCacheMap.cpp
+++ b/netwerk/cache/nsDiskCacheMap.cpp
@@ -176,22 +176,18 @@ nsDiskCacheMap::Open(nsIFile *  cacheDir
     // set dirty bit and flush header
     mHeader.mIsDirty    = true;
     rv = FlushHeader();
     if (NS_FAILED(rv)) {
         *corruptInfo = nsDiskCache::kFlushHeaderError;
         goto error_exit;
     }
     
-    {
-        // extra scope so the compiler doesn't barf on the above gotos jumping
-        // past this declaration down here
-        uint32_t overhead = moz_malloc_size_of(mRecordArray);
-        Telemetry::Accumulate(Telemetry::HTTP_DISK_CACHE_OVERHEAD, overhead);
-    }
+    Telemetry::Accumulate(Telemetry::HTTP_DISK_CACHE_OVERHEAD,
+                          (uint32_t)SizeOfExcludingThis(moz_malloc_size_of));
 
     *corruptInfo = nsDiskCache::kNotCorrupt;
     return NS_OK;
     
 error_exit:
     (void) Close(false);
        
     return rv;
@@ -1212,16 +1208,34 @@ nsDiskCacheMap::NotifyCapacityChange(uin
   const int32_t RECORD_COUNT_LIMIT = 32 * 1024 * 1024 / sizeof(nsDiskCacheRecord);
   int32_t maxRecordCount = NS_MIN(int32_t(capacity), RECORD_COUNT_LIMIT);
   if (mMaxRecordCount < maxRecordCount) {
     // We can only grow
     mMaxRecordCount = maxRecordCount;
   }
 }
 
+size_t
+nsDiskCacheMap::SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf)
+{
+  size_t usage = aMallocSizeOf(mRecordArray);
+
+  usage += aMallocSizeOf(mBuffer);
+  usage += aMallocSizeOf(mMapFD);
+  usage += aMallocSizeOf(mCleanFD);
+  usage += aMallocSizeOf(mCacheDirectory);
+  usage += aMallocSizeOf(mCleanCacheTimer);
+
+  for (int i = 0; i < kNumBlockFiles; i++) {
+    usage += mBlockFile[i].SizeOfExcludingThis(aMallocSizeOf);
+  }
+
+  return usage;
+}
+
 nsresult
 nsDiskCacheMap::InitCacheClean(nsIFile *  cacheDirectory,
                                nsDiskCache::CorruptCacheInfo *  corruptInfo,
                                bool reportCacheCleanTelemetryData)
 {
     // The _CACHE_CLEAN_ file will be used in the future to determine
     // if the cache is clean or not. 
     bool cacheCleanFileExists = false;
--- a/netwerk/cache/nsDiskCacheMap.h
+++ b/netwerk/cache/nsDiskCacheMap.h
@@ -480,16 +480,18 @@ public:
                 // Round up to nearest K
                 DecrementTotalSize(((blocks*blockSize) + 0x03FF) >> 10);
              }
                  
     uint32_t TotalSize()   { return mHeader.mDataSize; }
     
     int32_t  EntryCount()  { return mHeader.mEntryCount; }
 
+    size_t  SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf);
+
 
 private:
 
     /**
      *  Private methods
      */
     nsresult    OpenBlockFiles(nsDiskCache::CorruptCacheInfo *  corruptInfo);
     nsresult    CloseBlockFiles(bool flush);
--- a/netwerk/cache/nsDiskCacheStreams.cpp
+++ b/netwerk/cache/nsDiskCacheStreams.cpp
@@ -729,8 +729,20 @@ nsDiskCacheStreamIO::DeleteBuffer()
         NS_ASSERTION(!mBufDirty, "deleting dirty buffer");
         free(mBuffer);
         mBuffer = nullptr;
         mBufPos = 0;
         mBufEnd = 0;
         mBufSize = 0;
     }
 }
+
+size_t
+nsDiskCacheStreamIO::SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf)
+{
+    size_t usage = aMallocSizeOf(this);
+
+    usage += aMallocSizeOf(mLocalFile);
+    usage += aMallocSizeOf(mFD);
+    usage += aMallocSizeOf(mBuffer);
+
+    return usage;
+}
--- a/netwerk/cache/nsDiskCacheStreams.h
+++ b/netwerk/cache/nsDiskCacheStreams.h
@@ -41,16 +41,18 @@ public:
     
     void        IncrementInputStreamCount() { PR_ATOMIC_INCREMENT(&mInStreamCount); }
     void        DecrementInputStreamCount()
                 {
                     PR_ATOMIC_DECREMENT(&mInStreamCount);
                     NS_ASSERTION(mInStreamCount >= 0, "mInStreamCount has gone negative");
                 }
 
+    size_t     SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf);
+
     // GCC 2.95.2 requires this to be defined, although we never call it.
     // and OS/2 requires that it not be private
     nsDiskCacheStreamIO() { NS_NOTREACHED("oops"); }
 private:
     nsresult    OpenCacheFile(int flags, PRFileDesc ** fd);
     nsresult    ReadCacheBlocks();
     nsresult    FlushBufferToFile();
     void        UpdateFileSize();
--- a/toolkit/components/telemetry/Histograms.json
+++ b/toolkit/components/telemetry/Histograms.json
@@ -1092,16 +1092,22 @@
     "description": "Time spent waiting on the cache service lock (ms) on the main thread in NSCACHESERVICE_GETCACHEIOTARGET"
   },
   "CACHE_SERVICE_LOCK_WAIT_MAINTHREAD_NSCACHESERVICE_EVICTENTRIESFORCLIENT": {
     "kind": "exponential",
     "high": "10 * 1000",
     "n_buckets": 50,
     "description": "Time spent waiting on the cache service lock (ms) on the main thread in NSCACHESERVICE_EVICTENTRIESFORCLIENT"
   },
+  "CACHE_SERVICE_LOCK_WAIT_MAINTHREAD_NSCACHESERVICE_DISKDEVICEHEAPSIZE": {
+    "kind": "exponential",
+    "high": "10 * 1000",
+    "n_buckets": 50,
+    "description": "Time spent waiting on the cache service lock (ms) on the main thread in NSCACHESERVICE_DISKDEVICEHEAPSIZE"
+  },
   "CACHE_SERVICE_LOCK_WAIT_MAINTHREAD_NSCACHEENTRYDESCRIPTOR_DOOM": {
     "kind": "exponential",
     "high": "10 * 1000",
     "n_buckets": 50,
     "description": "Time spent waiting on the cache service lock (ms) on the main thread in NSCACHEENTRYDESCRIPTOR_DOOM"
   },
   "CACHE_SERVICE_LOCK_WAIT_MAINTHREAD_NSCACHEENTRYDESCRIPTOR_SETPREDICTEDDATASIZE": {
     "kind": "exponential",