Bug 898558 - Rework the jemalloc heap memory reporters. r=njn
authorJustin Lebar <justin.lebar@gmail.com>
Mon, 29 Jul 2013 09:10:53 -0700
changeset 152699 d4f1b39a85750b84a792920bcd4f4714cb574ed5
parent 152698 905805f269ba4af34a7dc8096a61c56a36623ddf
child 152700 268f7f146100cf9ece5af8ccc399e0625c4ac973
push id2859
push userakeybl@mozilla.com
push dateMon, 16 Sep 2013 19:14:59 +0000
treeherdermozilla-beta@87d3c51cd2bf [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnjn
bugs898558
milestone25.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 898558 - Rework the jemalloc heap memory reporters. r=njn Now we have the following hierarchy explicit/ heap-overhead/ bookkeeping waste page-cache (previosuly known as dirty) In addition, heap-committed-unused-ratio is now known as heap-overhead-ratio.
toolkit/components/aboutmemory/content/aboutMemory.js
toolkit/components/telemetry/TelemetryPing.js
xpcom/base/nsMemoryReporterManager.cpp
--- a/toolkit/components/aboutmemory/content/aboutMemory.js
+++ b/toolkit/components/aboutmemory/content/aboutMemory.js
@@ -315,29 +315,33 @@ function appendElementWithText(aP, aTagN
 }
 
 //---------------------------------------------------------------------------
 // Code specific to about:memory
 //---------------------------------------------------------------------------
 
 const kTreeDescriptions = {
   'explicit' :
-"This tree covers explicit memory allocations by the application, both at the \
-operating system level (via calls to functions such as VirtualAlloc, \
-vm_allocate, and mmap), and at the heap allocation level (via functions such \
-as malloc, calloc, realloc, memalign, operator new, and operator new[]) that \
-have not been explicitly decommitted (i.e. evicted from memory and swap). \
+"This tree covers explicit memory allocations by the application.  It includes \
+\n\n\
+* allocations made at the operating system level (via calls to functions such as \
+VirtualAlloc, vm_allocate, and mmap), \
+\n\n\
+* allocations made at the heap allocation level (via functions such as malloc, \
+calloc, realloc, memalign, operator new, and operator new[]) that have not been \
+explicitly decommitted (i.e. evicted from memory and swap), and \
+\n\n\
+* where possible, the overhead of the heap allocator itself.\
 \n\n\
 It excludes memory that is mapped implicitly such as code and data segments, \
-and thread stacks.  It also excludes heap memory that has been freed by the \
-application but is still being held onto by the heap allocator. \
+and thread stacks. \
 \n\n\
-It is not guaranteed to cover every explicit allocation, but it does cover \
-most (including the entire heap), and therefore it is the single best number \
-to focus on when trying to reduce memory usage.",
+'explicit' is not guaranteed to cover every explicit allocation, but it does cover \
+most (including the entire heap), and therefore it is the single best number to \
+focus on when trying to reduce memory usage.",
 
   'rss':
 "This tree shows how much space in physical memory each of the process's \
 mappings is currently using (the mapping's 'resident set size', or 'RSS'). \
 This is a good measure of the 'cost' of the mapping, although it does not \
 take into account the fact that shared libraries may be mapped by multiple \
 processes but appear only once in physical memory. \
 \n\n\
--- a/toolkit/components/telemetry/TelemetryPing.js
+++ b/toolkit/components/telemetry/TelemetryPing.js
@@ -67,18 +67,18 @@ const MEM_HISTOGRAMS = {
   "js-compartments/user": "MEMORY_JS_COMPARTMENTS_USER",
   "js-main-runtime-temporary-peak": "MEMORY_JS_MAIN_RUNTIME_TEMPORARY_PEAK",
   "resident-fast": "MEMORY_RESIDENT",
   "vsize": "MEMORY_VSIZE",
   "storage-sqlite": "MEMORY_STORAGE_SQLITE",
   "images-content-used-uncompressed":
     "MEMORY_IMAGES_CONTENT_USED_UNCOMPRESSED",
   "heap-allocated": "MEMORY_HEAP_ALLOCATED",
-  "heap-committed-unused": "MEMORY_HEAP_COMMITTED_UNUSED",
-  "heap-committed-unused-ratio": "MEMORY_HEAP_COMMITTED_UNUSED_RATIO",
+  "heap-overhead": "MEMORY_HEAP_COMMITTED_UNUSED",
+  "heap-overhead-ratio": "MEMORY_HEAP_COMMITTED_UNUSED_RATIO",
   "page-faults-hard": "PAGE_FAULTS_HARD",
   "low-memory-events/virtual": "LOW_MEMORY_EVENTS_VIRTUAL",
   "low-memory-events/physical": "LOW_MEMORY_EVENTS_PHYSICAL",
   "ghost-windows": "GHOST_WINDOWS"
 };
 
 // Seconds of idle time before pinging.
 // On idle-daily a gather-telemetry notification is fired, during it probes can
--- a/xpcom/base/nsMemoryReporterManager.cpp
+++ b/xpcom/base/nsMemoryReporterManager.cpp
@@ -474,113 +474,16 @@ public:
 /**
  ** memory reporter implementation for jemalloc and OSX malloc,
  ** to obtain info on total memory in use (that we know about,
  ** at least -- on OSX, there are sometimes other zones in use).
  **/
 
 #ifdef HAVE_JEMALLOC_STATS
 
-class HeapCommittedReporter MOZ_FINAL : public MemoryReporterBase
-{
-public:
-    HeapCommittedReporter()
-      : MemoryReporterBase("heap-committed", KIND_OTHER, UNITS_BYTES,
-"Memory mapped by the heap allocator that is committed, i.e. in physical "
-"memory or paged to disk.  When 'heap-committed' is larger than "
-"'heap-allocated', the difference between the two values is likely due to "
-"external fragmentation; that is, the allocator allocated a large block of "
-"memory and is unable to decommit it because a small part of that block is "
-"currently in use.")
-    {}
-private:
-    int64_t Amount()
-    {
-        jemalloc_stats_t stats;
-        jemalloc_stats(&stats);
-        return (int64_t) stats.committed;
-    }
-};
-
-class HeapCommittedUnusedReporter MOZ_FINAL : public MemoryReporterBase
-{
-public:
-    HeapCommittedUnusedReporter()
-      : MemoryReporterBase("heap-committed-unused", KIND_OTHER, UNITS_BYTES,
-"Committed bytes which do not correspond to an active allocation; i.e., "
-"'heap-committed' - 'heap-allocated'.  Although the allocator will waste some "
-"space under any circumstances, a large value here may indicate that the "
-"heap is highly fragmented.")
-    {}
-private:
-    int64_t Amount()
-    {
-        jemalloc_stats_t stats;
-        jemalloc_stats(&stats);
-        return stats.committed - stats.allocated;
-    }
-};
-
-class HeapCommittedUnusedRatioReporter MOZ_FINAL : public MemoryReporterBase
-{
-public:
-    HeapCommittedUnusedRatioReporter()
-      : MemoryReporterBase("heap-committed-unused-ratio", KIND_OTHER,
-                           UNITS_PERCENTAGE,
-"Ratio of committed, unused bytes to allocated bytes; i.e., "
-"'heap-committed-unused' / 'heap-allocated'.  This measures the overhead of "
-"the heap allocator relative to amount of memory allocated.")
-    {}
-private:
-    int64_t Amount()
-    {
-        jemalloc_stats_t stats;
-        jemalloc_stats(&stats);
-        return (int64_t) 10000 * (stats.committed - stats.allocated) /
-                                  ((double)stats.allocated);
-    }
-};
-
-class HeapDirtyReporter MOZ_FINAL : public MemoryReporterBase
-{
-public:
-    HeapDirtyReporter()
-      : MemoryReporterBase("heap-dirty", KIND_OTHER, UNITS_BYTES,
-"Memory which the allocator could return to the operating system, but hasn't. "
-"The allocator keeps this memory around as an optimization, so it doesn't "
-"have to ask the OS the next time it needs to fulfill a request. This value "
-"is typically not larger than a few megabytes.")
-    {}
-private:
-    int64_t Amount()
-    {
-        jemalloc_stats_t stats;
-        jemalloc_stats(&stats);
-        return (int64_t) stats.dirty;
-    }
-};
-
-class HeapUnusedReporter MOZ_FINAL : public MemoryReporterBase
-{
-public:
-    HeapUnusedReporter()
-      : MemoryReporterBase("heap-unused", KIND_OTHER, UNITS_BYTES,
-"Memory mapped by the heap allocator that is not part of an active "
-"allocation. Much of this memory may be uncommitted -- that is, it does not "
-"take up space in physical memory or in the swap file.")
-    {}
-private:
-    int64_t Amount()
-    {
-        jemalloc_stats_t stats;
-        jemalloc_stats(&stats);
-        return (int64_t) (stats.mapped - stats.allocated);
-    }
-};
-
 class HeapAllocatedReporter MOZ_FINAL : public MemoryReporterBase
 {
 public:
     HeapAllocatedReporter()
       : MemoryReporterBase("heap-allocated", KIND_OTHER, UNITS_BYTES,
 "Memory mapped by the heap allocator that is currently allocated to the "
 "application.  This may exceed the amount of memory requested by the "
 "application because the allocator regularly rounds up request sizes. (The "
@@ -589,16 +492,119 @@ public:
 private:
     int64_t Amount()
     {
         jemalloc_stats_t stats;
         jemalloc_stats(&stats);
         return (int64_t) stats.allocated;
     }
 };
+
+class HeapOverheadWasteReporter MOZ_FINAL : public MemoryReporterBase
+{
+public:
+    // We mark this and the other heap-overhead reporters as KIND_NONHEAP
+    // because KIND_HEAP memory means "counted in heap-allocated", which this
+    // is not.
+    HeapOverheadWasteReporter()
+      : MemoryReporterBase("explicit/heap-overhead/waste",
+                           KIND_NONHEAP, UNITS_BYTES,
+"Committed bytes which do not correspond to an active allocation and which the "
+"allocator is not intentionally keeping alive (i.e., not 'heap-bookkeeping' or "
+"'heap-page-cache').  Although the allocator will waste some space under any "
+"circumstances, a large value here may indicate that the heap is highly "
+"fragmented, or that allocator is performing poorly for some other reason.")
+    {}
+private:
+    int64_t Amount()
+    {
+        jemalloc_stats_t stats;
+        jemalloc_stats(&stats);
+        return stats.waste;
+    }
+};
+
+class HeapOverheadBookkeepingReporter MOZ_FINAL : public MemoryReporterBase
+{
+public:
+    HeapOverheadBookkeepingReporter()
+      : MemoryReporterBase("explicit/heap-overhead/bookkeeping",
+                           KIND_NONHEAP, UNITS_BYTES,
+"Committed bytes which the heap allocator uses for internal data structures.")
+    {}
+private:
+    int64_t Amount()
+    {
+        jemalloc_stats_t stats;
+        jemalloc_stats(&stats);
+        return stats.bookkeeping;
+    }
+};
+
+class HeapOverheadPageCacheReporter MOZ_FINAL : public MemoryReporterBase
+{
+public:
+    HeapOverheadPageCacheReporter()
+      : MemoryReporterBase("explicit/heap-overhead/page-cache",
+                           KIND_NONHEAP, UNITS_BYTES,
+"Memory which the allocator could return to the operating system, but hasn't. "
+"The allocator keeps this memory around as an optimization, so it doesn't "
+"have to ask the OS the next time it needs to fulfill a request. This value "
+"is typically not larger than a few megabytes.")
+    {}
+private:
+    int64_t Amount()
+    {
+        jemalloc_stats_t stats;
+        jemalloc_stats(&stats);
+        return (int64_t) stats.page_cache;
+    }
+};
+
+class HeapCommittedReporter MOZ_FINAL : public MemoryReporterBase
+{
+public:
+    HeapCommittedReporter()
+      : MemoryReporterBase("heap-committed", KIND_OTHER, UNITS_BYTES,
+"Memory mapped by the heap allocator that is committed, i.e. in physical "
+"memory or paged to disk.  This value corresponds to 'heap-allocated' + "
+"'heap-waste' + 'heap-bookkeeping' + 'heap-page-cache', but because "
+"these values are read at different times, the result probably won't match "
+"exactly.")
+    {}
+private:
+    int64_t Amount()
+    {
+        jemalloc_stats_t stats;
+        jemalloc_stats(&stats);
+        return (int64_t) (stats.allocated + stats.waste +
+                          stats.bookkeeping + stats.page_cache);
+    }
+};
+
+class HeapOverheadRatioReporter MOZ_FINAL : public MemoryReporterBase
+{
+public:
+    HeapOverheadRatioReporter()
+      : MemoryReporterBase("heap-overhead-ratio", KIND_OTHER,
+                           UNITS_PERCENTAGE,
+"Ratio of committed, unused bytes to allocated bytes; i.e., "
+"'heap-overhead' / 'heap-allocated'.  This measures the overhead of "
+"the heap allocator relative to amount of memory allocated.")
+    {}
+private:
+    int64_t Amount()
+    {
+        jemalloc_stats_t stats;
+        jemalloc_stats(&stats);
+        return (int64_t) 10000 *
+          (stats.waste + stats.bookkeeping + stats.page_cache) /
+          ((double)stats.allocated);
+    }
+};
 #endif  // HAVE_JEMALLOC_STATS
 
 // Why is this here?  At first glance, you'd think it could be defined and
 // registered with nsMemoryReporterManager entirely within nsAtomTable.cpp.
 // However, the obvious time to register it is when the table is initialized,
 // and that happens before XPCOM components are initialized, which means the
 // NS_RegisterMemoryReporter call fails.  So instead we do it here.
 class AtomTablesReporter MOZ_FINAL : public MemoryReporterBase
@@ -689,21 +695,21 @@ nsMemoryReporterManager::Init()
 {
 #if defined(HAVE_JEMALLOC_STATS) && defined(XP_LINUX)
     if (!jemalloc_stats)
         return NS_ERROR_FAILURE;
 #endif
 
 #ifdef HAVE_JEMALLOC_STATS
     RegisterReporter(new HeapAllocatedReporter);
-    RegisterReporter(new HeapUnusedReporter);
+    RegisterReporter(new HeapOverheadWasteReporter);
+    RegisterReporter(new HeapOverheadBookkeepingReporter);
+    RegisterReporter(new HeapOverheadPageCacheReporter);
     RegisterReporter(new HeapCommittedReporter);
-    RegisterReporter(new HeapCommittedUnusedReporter);
-    RegisterReporter(new HeapCommittedUnusedRatioReporter);
-    RegisterReporter(new HeapDirtyReporter);
+    RegisterReporter(new HeapOverheadRatioReporter);
 #endif
 
 #ifdef HAVE_VSIZE_AND_RESIDENT_REPORTERS
     RegisterReporter(new VsizeReporter);
     RegisterReporter(new ResidentReporter);
     RegisterReporter(new ResidentFastReporter);
 #endif