Bug 699951 - add a memory reporter for heap usage by the disk cache. r=michal
authorNick Hurley <hurley@todesschaf.org>
Mon, 05 Nov 2012 10:22:33 -0800
changeset 120271 c87fcf6ed0a77d0f532b41217b584a806879c5b9
parent 120270 c999fc311c2859a4f35475a7745317bead62a2e7
child 120272 0dea4f9a91330ce98542cb2d131e385e39d3c878
push id1997
push userakeybl@mozilla.com
push dateMon, 07 Jan 2013 21:25:26 +0000
treeherdermozilla-beta@4baf45cdcf21 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmichal
bugs699951
milestone19.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 699951 - add a memory reporter for heap usage by the disk cache. r=michal
netwerk/cache/nsCacheService.cpp
netwerk/cache/nsCacheService.h
netwerk/cache/nsDiskCacheBinding.cpp
netwerk/cache/nsDiskCacheBinding.h
netwerk/cache/nsDiskCacheBlockFile.cpp
netwerk/cache/nsDiskCacheBlockFile.h
netwerk/cache/nsDiskCacheDevice.cpp
netwerk/cache/nsDiskCacheDevice.h
netwerk/cache/nsDiskCacheMap.cpp
netwerk/cache/nsDiskCacheMap.h
netwerk/cache/nsDiskCacheStreams.cpp
netwerk/cache/nsDiskCacheStreams.h
toolkit/components/telemetry/Histograms.json
--- a/netwerk/cache/nsCacheService.cpp
+++ b/netwerk/cache/nsCacheService.cpp
@@ -1055,22 +1055,33 @@ private:
 /******************************************************************************
  * nsCacheService
  *****************************************************************************/
 nsCacheService *   nsCacheService::gService = nullptr;
 
 static nsCOMPtr<nsIMemoryReporter> MemoryCacheReporter = nullptr;
 
 NS_THREADSAFE_MEMORY_REPORTER_IMPLEMENT(NetworkMemoryCache,
-    "explicit/network-memory-cache",
+    "explicit/network/memory-cache",
     KIND_HEAP,
     UNITS_BYTES,
     nsCacheService::MemoryDeviceSize,
     "Memory used by the network memory cache.")
 
+NS_MEMORY_REPORTER_MALLOC_SIZEOF_FUN(NetworkDiskCacheSizeOfFun, "network-disk-cache")
+
+static nsCOMPtr<nsIMemoryReporter> DiskCacheReporter = nullptr;
+
+NS_THREADSAFE_MEMORY_REPORTER_IMPLEMENT(NetworkDiskCache,
+    "explicit/network/disk-cache",
+    KIND_HEAP,
+    UNITS_BYTES,
+    nsCacheService::DiskDeviceHeapSize,
+    "Memory used by the network disk cache.")
+
 NS_IMPL_THREADSAFE_ISUPPORTS1(nsCacheService, nsICacheService)
 
 nsCacheService::nsCacheService()
     : mLock("nsCacheService.mLock"),
       mCondVar(mLock, "nsCacheService.mCondVar"),
       mInitialized(false),
       mClearingEntries(false),
       mEnableMemoryDevice(true),
@@ -1201,21 +1212,24 @@ nsCacheService::Shutdown()
         (void) SyncWithCacheIOThread();
 
         // obtain the disk cache directory in case we need to sanitize it
         parentDir = mObserver->DiskCacheParentDirectory();
         shouldSanitize = mObserver->SanitizeAtShutdown();
         mObserver->Remove();
         NS_RELEASE(mObserver);
         
-        // unregister memory reporter, before deleting the memory device, just
+        // unregister memory reporters, before deleting the devices, just
         // to be safe
         NS_UnregisterMemoryReporter(MemoryCacheReporter);
         MemoryCacheReporter = nullptr;
 
+        NS_UnregisterMemoryReporter(DiskCacheReporter);
+        DiskCacheReporter = nullptr;
+
         // deallocate memory and disk caches
         delete mMemoryDevice;
         mMemoryDevice = nullptr;
 
         delete mDiskDevice;
         mDiskDevice = nullptr;
 
         if (mOfflineDevice)
@@ -1559,16 +1573,19 @@ nsCacheService::CreateDiskDevice()
             mSmartSizeTimer = nullptr;
         }
     } else {
         NS_WARNING("Can't create smart size timer");
     }
     // Ignore state of the timer and return success since the purpose of the
     // method (create the disk-device) has been fulfilled
 
+    DiskCacheReporter = new NS_MEMORY_REPORTER_NAME(NetworkDiskCache);
+    NS_RegisterMemoryReporter(DiskCacheReporter);
+
     return NS_OK;
 }
 
 // Runnable sent from cache thread to main thread
 class nsDisableOldMaxSmartSizePrefEvent: public nsRunnable
 {
 public:
     nsDisableOldMaxSmartSizePrefEvent() {}
@@ -2231,16 +2248,24 @@ nsCacheService::EnsureEntryHasDevice(nsC
 
 int64_t
 nsCacheService::MemoryDeviceSize()
 {
     nsMemoryCacheDevice *memoryDevice = GlobalInstance()->mMemoryDevice;
     return memoryDevice ? memoryDevice->TotalSize() : 0;
 }
 
+int64_t
+nsCacheService::DiskDeviceHeapSize()
+{
+    nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_DISKDEVICEHEAPSIZE));
+    nsDiskCacheDevice *diskDevice = GlobalInstance()->mDiskDevice;
+    return (int64_t)(diskDevice ? diskDevice->SizeOfIncludingThis(NetworkDiskCacheSizeOfFun) : 0);
+}
+
 nsresult
 nsCacheService::DoomEntry(nsCacheEntry * entry)
 {
     return gService->DoomEntry_Internal(entry, true);
 }
 
 
 nsresult
--- a/netwerk/cache/nsCacheService.h
+++ b/netwerk/cache/nsCacheService.h
@@ -124,16 +124,18 @@ public:
     /**
      * Methods called by any cache classes
      */
 
     static
     nsCacheService * GlobalInstance()   { return gService; }
 
     static int64_t   MemoryDeviceSize();
+
+    static int64_t   DiskDeviceHeapSize();
     
     static nsresult  DoomEntry(nsCacheEntry * entry);
 
     static bool      IsStorageEnabledForPolicy_Locked(nsCacheStoragePolicy policy);
 
     /**
      * Called by disk cache to notify us to use the new max smart size
      */
--- a/netwerk/cache/nsDiskCacheBinding.cpp
+++ b/netwerk/cache/nsDiskCacheBinding.cpp
@@ -363,8 +363,60 @@ nsDiskCacheBindery::ActiveBindings()
     NS_ASSERTION(initialized, "nsDiskCacheBindery not initialized");
     if (!initialized) return false;
 
     bool    activeBinding = false;
     PL_DHashTableEnumerate(&table, ActiveBinding, &activeBinding);
 
     return activeBinding;
 }
+
+struct AccumulatorArg {
+    size_t mUsage;
+    nsMallocSizeOfFun mMallocSizeOf;
+};
+
+PLDHashOperator
+AccumulateHeapUsage(PLDHashTable *table, PLDHashEntryHdr *hdr, uint32_t number,
+                    void *arg)
+{
+    nsDiskCacheBinding *binding = ((HashTableEntry *)hdr)->mBinding;
+    NS_ASSERTION(binding, "### disk cache binding = nsnull!");
+
+    AccumulatorArg *acc = (AccumulatorArg *)arg;
+
+    nsDiskCacheBinding *head = binding;
+    do {
+        acc->mUsage += acc->mMallocSizeOf(binding);
+
+        if (binding->mStreamIO) {
+            acc->mUsage += binding->mStreamIO->SizeOfIncludingThis(acc->mMallocSizeOf);
+        }
+
+        /* No good way to get at mDeactivateEvent internals for proper size, so
+           we use this as an estimate. */
+        if (binding->mDeactivateEvent) {
+            acc->mUsage += acc->mMallocSizeOf(binding->mDeactivateEvent);
+        }
+
+        binding = (nsDiskCacheBinding *)PR_NEXT_LINK(binding);
+    } while (binding != head);
+
+    return PL_DHASH_NEXT;
+}
+
+/**
+ * SizeOfExcludingThis: return the amount of heap memory (bytes) being used by the bindery
+ */
+size_t
+nsDiskCacheBindery::SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf)
+{
+    NS_ASSERTION(initialized, "nsDiskCacheBindery not initialized");
+    if (!initialized) return 0;
+
+    AccumulatorArg arg;
+    arg.mUsage = 0;
+    arg.mMallocSizeOf = aMallocSizeOf;
+
+    PL_DHashTableEnumerate(&table, AccumulateHeapUsage, &arg);
+
+    return arg.mUsage;
+}
--- a/netwerk/cache/nsDiskCacheBinding.h
+++ b/netwerk/cache/nsDiskCacheBinding.h
@@ -99,17 +99,19 @@ public:
     void                    Reset();
 
     nsDiskCacheBinding *    CreateBinding(nsCacheEntry *       entry,
                                           nsDiskCacheRecord *  record);
 
     nsDiskCacheBinding *    FindActiveBinding(uint32_t  hashNumber);
     void                    RemoveBinding(nsDiskCacheBinding * binding);
     bool                    ActiveBindings();
-    
+
+    size_t                 SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf);
+
 private:
     nsresult                AddBinding(nsDiskCacheBinding * binding);
 
     // member variables
     static PLDHashTableOps ops;
     PLDHashTable           table;
     bool                   initialized;
 };
--- a/netwerk/cache/nsDiskCacheBlockFile.cpp
+++ b/netwerk/cache/nsDiskCacheBlockFile.cpp
@@ -388,8 +388,14 @@ nsDiskCacheBlockFile::Write(int32_t offs
         mFileSize = NS_MIN(mFileSize, maxFileSize);
         //  Appears to cause bug 617123?  Disabled for now.
         //mozilla::fallocate(mFD, mFileSize);
     }
     if (PR_Seek(mFD, offset, PR_SEEK_SET) != offset)
         return false;
     return PR_Write(mFD, buf, amount) == amount;
 }
+
+size_t
+nsDiskCacheBlockFile::SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf)
+{
+    return aMallocSizeOf(mBitMap) + aMallocSizeOf(mFD);
+}
--- a/netwerk/cache/nsDiskCacheBlockFile.h
+++ b/netwerk/cache/nsDiskCacheBlockFile.h
@@ -39,17 +39,19 @@ public:
      * Truncates the block file to the end of the last allocated block.
      */
     nsresult  Trim() { return nsDiskCache::Truncate(mFD, CalcBlockFileSize()); }
     nsresult  DeallocateBlocks( int32_t  startBlock, int32_t  numBlocks);
     nsresult  WriteBlocks( void * buffer, uint32_t size, int32_t  numBlocks, 
                            int32_t * startBlock);
     nsresult  ReadBlocks( void * buffer, int32_t  startBlock, int32_t  numBlocks, 
                           int32_t * bytesRead);
-    
+
+    size_t   SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf);
+
 private:
     nsresult  FlushBitMap();
     int32_t   AllocateBlocks( int32_t  numBlocks);
     nsresult  VerifyAllocation( int32_t startBlock, int32_t numBLocks);
     uint32_t  CalcBlockFileSize();
     bool   Write(int32_t offset, const void *buf, int32_t amount);
 
 /**
--- a/netwerk/cache/nsDiskCacheDevice.cpp
+++ b/netwerk/cache/nsDiskCacheDevice.cpp
@@ -1155,8 +1155,20 @@ nsDiskCacheDevice::SetMaxEntrySize(int32
 {
     // Internal units are bytes. Changing this only takes effect *after* the
     // change and has no consequences for existing cache-entries
     if (maxSizeInKilobytes >= 0)
         mMaxEntrySize = maxSizeInKilobytes * 1024;
     else
         mMaxEntrySize = -1;
 }
+
+size_t
+nsDiskCacheDevice::SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf)
+{
+    size_t usage = aMallocSizeOf(this);
+
+    usage += mCacheMap.SizeOfExcludingThis(aMallocSizeOf);
+    usage += mBindery.SizeOfExcludingThis(aMallocSizeOf);
+
+    return usage;
+}
+
--- a/netwerk/cache/nsDiskCacheDevice.h
+++ b/netwerk/cache/nsDiskCacheDevice.h
@@ -49,16 +49,18 @@ public:
     virtual nsresult        OnDataSizeChange(nsCacheEntry * entry, int32_t deltaSize);
     
     virtual nsresult        Visit(nsICacheVisitor * visitor);
 
     virtual nsresult        EvictEntries(const char * clientID);
 
     bool                    EntryIsTooBig(int64_t entrySize);
 
+    size_t                 SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf);
+
     /**
      * Preference accessors
      */
     void                    SetCacheParentDirectory(nsIFile * parentDir);
     void                    SetCapacity(uint32_t  capacity);
     void                    SetMaxEntrySize(int32_t  maxSizeInKilobytes);
 
 /* private: */
--- a/netwerk/cache/nsDiskCacheMap.cpp
+++ b/netwerk/cache/nsDiskCacheMap.cpp
@@ -176,22 +176,18 @@ nsDiskCacheMap::Open(nsIFile *  cacheDir
     // set dirty bit and flush header
     mHeader.mIsDirty    = true;
     rv = FlushHeader();
     if (NS_FAILED(rv)) {
         *corruptInfo = nsDiskCache::kFlushHeaderError;
         goto error_exit;
     }
     
-    {
-        // extra scope so the compiler doesn't barf on the above gotos jumping
-        // past this declaration down here
-        uint32_t overhead = moz_malloc_size_of(mRecordArray);
-        Telemetry::Accumulate(Telemetry::HTTP_DISK_CACHE_OVERHEAD, overhead);
-    }
+    Telemetry::Accumulate(Telemetry::HTTP_DISK_CACHE_OVERHEAD,
+                          (uint32_t)SizeOfExcludingThis(moz_malloc_size_of));
 
     *corruptInfo = nsDiskCache::kNotCorrupt;
     return NS_OK;
     
 error_exit:
     (void) Close(false);
        
     return rv;
@@ -1212,16 +1208,34 @@ nsDiskCacheMap::NotifyCapacityChange(uin
   const int32_t RECORD_COUNT_LIMIT = 32 * 1024 * 1024 / sizeof(nsDiskCacheRecord);
   int32_t maxRecordCount = NS_MIN(int32_t(capacity), RECORD_COUNT_LIMIT);
   if (mMaxRecordCount < maxRecordCount) {
     // We can only grow
     mMaxRecordCount = maxRecordCount;
   }
 }
 
+size_t
+nsDiskCacheMap::SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf)
+{
+  size_t usage = aMallocSizeOf(mRecordArray);
+
+  usage += aMallocSizeOf(mBuffer);
+  usage += aMallocSizeOf(mMapFD);
+  usage += aMallocSizeOf(mCleanFD);
+  usage += aMallocSizeOf(mCacheDirectory);
+  usage += aMallocSizeOf(mCleanCacheTimer);
+
+  for (int i = 0; i < kNumBlockFiles; i++) {
+    usage += mBlockFile[i].SizeOfExcludingThis(aMallocSizeOf);
+  }
+
+  return usage;
+}
+
 nsresult
 nsDiskCacheMap::InitCacheClean(nsIFile *  cacheDirectory,
                                nsDiskCache::CorruptCacheInfo *  corruptInfo,
                                bool reportCacheCleanTelemetryData)
 {
     // The _CACHE_CLEAN_ file will be used in the future to determine
     // if the cache is clean or not. 
     bool cacheCleanFileExists = false;
--- a/netwerk/cache/nsDiskCacheMap.h
+++ b/netwerk/cache/nsDiskCacheMap.h
@@ -480,16 +480,18 @@ public:
                 // Round up to nearest K
                 DecrementTotalSize(((blocks*blockSize) + 0x03FF) >> 10);
              }
                  
     uint32_t TotalSize()   { return mHeader.mDataSize; }
     
     int32_t  EntryCount()  { return mHeader.mEntryCount; }
 
+    size_t  SizeOfExcludingThis(nsMallocSizeOfFun aMallocSizeOf);
+
 
 private:
 
     /**
      *  Private methods
      */
     nsresult    OpenBlockFiles(nsDiskCache::CorruptCacheInfo *  corruptInfo);
     nsresult    CloseBlockFiles(bool flush);
--- a/netwerk/cache/nsDiskCacheStreams.cpp
+++ b/netwerk/cache/nsDiskCacheStreams.cpp
@@ -729,8 +729,20 @@ nsDiskCacheStreamIO::DeleteBuffer()
         NS_ASSERTION(!mBufDirty, "deleting dirty buffer");
         free(mBuffer);
         mBuffer = nullptr;
         mBufPos = 0;
         mBufEnd = 0;
         mBufSize = 0;
     }
 }
+
+size_t
+nsDiskCacheStreamIO::SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf)
+{
+    size_t usage = aMallocSizeOf(this);
+
+    usage += aMallocSizeOf(mLocalFile);
+    usage += aMallocSizeOf(mFD);
+    usage += aMallocSizeOf(mBuffer);
+
+    return usage;
+}
--- a/netwerk/cache/nsDiskCacheStreams.h
+++ b/netwerk/cache/nsDiskCacheStreams.h
@@ -41,16 +41,18 @@ public:
     
     void        IncrementInputStreamCount() { PR_ATOMIC_INCREMENT(&mInStreamCount); }
     void        DecrementInputStreamCount()
                 {
                     PR_ATOMIC_DECREMENT(&mInStreamCount);
                     NS_ASSERTION(mInStreamCount >= 0, "mInStreamCount has gone negative");
                 }
 
+    size_t     SizeOfIncludingThis(nsMallocSizeOfFun aMallocSizeOf);
+
     // GCC 2.95.2 requires this to be defined, although we never call it.
     // and OS/2 requires that it not be private
     nsDiskCacheStreamIO() { NS_NOTREACHED("oops"); }
 private:
     nsresult    OpenCacheFile(int flags, PRFileDesc ** fd);
     nsresult    ReadCacheBlocks();
     nsresult    FlushBufferToFile();
     void        UpdateFileSize();
--- a/toolkit/components/telemetry/Histograms.json
+++ b/toolkit/components/telemetry/Histograms.json
@@ -1092,16 +1092,22 @@
     "description": "Time spent waiting on the cache service lock (ms) on the main thread in NSCACHESERVICE_GETCACHEIOTARGET"
   },
   "CACHE_SERVICE_LOCK_WAIT_MAINTHREAD_NSCACHESERVICE_EVICTENTRIESFORCLIENT": {
     "kind": "exponential",
     "high": "10 * 1000",
     "n_buckets": 50,
     "description": "Time spent waiting on the cache service lock (ms) on the main thread in NSCACHESERVICE_EVICTENTRIESFORCLIENT"
   },
+  "CACHE_SERVICE_LOCK_WAIT_MAINTHREAD_NSCACHESERVICE_DISKDEVICEHEAPSIZE": {
+    "kind": "exponential",
+    "high": "10 * 1000",
+    "n_buckets": 50,
+    "description": "Time spent waiting on the cache service lock (ms) on the main thread in NSCACHESERVICE_DISKDEVICEHEAPSIZE"
+  },
   "CACHE_SERVICE_LOCK_WAIT_MAINTHREAD_NSCACHEENTRYDESCRIPTOR_DOOM": {
     "kind": "exponential",
     "high": "10 * 1000",
     "n_buckets": 50,
     "description": "Time spent waiting on the cache service lock (ms) on the main thread in NSCACHEENTRYDESCRIPTOR_DOOM"
   },
   "CACHE_SERVICE_LOCK_WAIT_MAINTHREAD_NSCACHEENTRYDESCRIPTOR_SETPREDICTEDDATASIZE": {
     "kind": "exponential",