Bug 1252375 - Fix up jemalloc stats reporting. r=erahm.
authorNicholas Nethercote <nnethercote@mozilla.com>
Thu, 03 Mar 2016 12:49:27 +1100
changeset 286750 90fae59aad6e801ca8c4c7226be35b7116c34d60
parent 286749 39482a4423f12087db7046d05f626fae2b7e2ebd
child 286751 530047c97d83eda3e7ca2e5c7a1d8bb5460cfc8d
push id18000
push usercbook@mozilla.com
push dateFri, 04 Mar 2016 12:40:23 +0000
treeherderfx-team@365dff9e6e1f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerserahm
bugs1252375
milestone47.0a1
Bug 1252375 - Fix up jemalloc stats reporting. r=erahm. We have some oddities in our jemalloc stats reporting. - "heap-overhead-ratio" is a strange measurement: overhead / non-overhead, expressed as a percentage. And it omits "bin_unused", which appears to be an oversight. - "heap-committed" also omits "bin_unused". - There are some minor errors in memory report descriptions. This patch fixes these and improves the heap reporting. It makes the following reporting changes: - "heap-allocated": Duplicated as "heap-committed/allocated". (We keep "heap-allocated" because that's a special value used in the computation of "heap-unclassified".) - "heap-committed/overhead": Added; it's the same as the sum of the "explicit/heap-overhead/*" values. Together with "heap-committed/allocated" it shows clearly what fraction of the heap is overhead and what fraction is useful. - "heap-committed": Removed; now implicit as the "heap-committed/" node. - "heap-overhead-ratio": - Removed from memory reports; now shown as the percentage of the new "heap-committed/overhead" node. - Still available as a distinguished amount (because it's useful in isolation) but renamed to heapOverheadFraction, and the telemetry ID is renamed as MEMORY_HEAP_OVERHEAD_FRACTION. - "heap-chunks": Removed; it's not that interesting, and can be manually computed as "heap-mapped" / "heap-chunksize" if necessary.
toolkit/components/aboutmemory/tests/test_memoryReporters.xul
toolkit/components/telemetry/Histograms.json
toolkit/components/telemetry/TelemetrySession.jsm
xpcom/base/nsIMemoryReporter.idl
xpcom/base/nsMemoryReporterManager.cpp
--- a/toolkit/components/aboutmemory/tests/test_memoryReporters.xul
+++ b/toolkit/components/aboutmemory/tests/test_memoryReporters.xul
@@ -146,17 +146,17 @@
   let amounts = [
     "vsize",
     "vsizeMaxContiguous",
     "resident",
     "residentFast",
     "residentPeak",
     "residentUnique",
     "heapAllocated",
-    "heapOverheadRatio",
+    "heapOverheadFraction",
     "JSMainRuntimeGCHeap",
     "JSMainRuntimeTemporaryPeak",
     "JSMainRuntimeCompartmentsSystem",
     "JSMainRuntimeCompartmentsUser",
     "imagesContentUsedUncompressed",
     "storageSQLite",
     "lowMemoryEventsVirtual",
     "lowMemoryEventsPhysical",
--- a/toolkit/components/telemetry/Histograms.json
+++ b/toolkit/components/telemetry/Histograms.json
@@ -658,23 +658,24 @@
     "alert_emails": ["memshrink-telemetry-alerts@mozilla.com"],
     "expires_in_version": "never",
     "kind": "exponential",
     "low": 1024,
     "high": 524288,
     "n_buckets": 50,
     "description": "Committed, unused heap memory (KB)"
   },
-  "MEMORY_HEAP_COMMITTED_UNUSED_RATIO": {
+  "MEMORY_HEAP_OVERHEAD_FRACTION": {
     "alert_emails": ["memshrink-telemetry-alerts@mozilla.com"],
+    "bug_numbers": [1252375],
     "expires_in_version": "never",
     "kind": "linear",
     "high": 100,
     "n_buckets": 25,
-    "description": "Ratio of committed, unused memory to allocated memory in the heap (percentage)."
+    "description": "Fraction of committed heap memory that is overhead (percentage)."
   },
   "GHOST_WINDOWS": {
     "alert_emails": ["memshrink-telemetry-alerts@mozilla.com"],
     "expires_in_version": "never",
     "kind": "exponential",
     "high": 128,
     "n_buckets": 32,
     "description": "Number of ghost windows"
--- a/toolkit/components/telemetry/TelemetrySession.jsm
+++ b/toolkit/components/telemetry/TelemetrySession.jsm
@@ -1072,17 +1072,17 @@ var Impl = {
     let cc= (id, n) => h(id, Ci.nsIMemoryReporter.UNITS_COUNT_CUMULATIVE, n);
     let p = (id, n) => h(id, Ci.nsIMemoryReporter.UNITS_PERCENTAGE, n);
 
     b("MEMORY_VSIZE", "vsize");
     b("MEMORY_VSIZE_MAX_CONTIGUOUS", "vsizeMaxContiguous");
     b("MEMORY_RESIDENT_FAST", "residentFast");
     b("MEMORY_UNIQUE", "residentUnique");
     b("MEMORY_HEAP_ALLOCATED", "heapAllocated");
-    p("MEMORY_HEAP_COMMITTED_UNUSED_RATIO", "heapOverheadRatio");
+    p("MEMORY_HEAP_OVERHEAD_FRACTION", "heapOverheadFraction");
     b("MEMORY_JS_GC_HEAP", "JSMainRuntimeGCHeap");
     b("MEMORY_JS_MAIN_RUNTIME_TEMPORARY_PEAK", "JSMainRuntimeTemporaryPeak");
     c("MEMORY_JS_COMPARTMENTS_SYSTEM", "JSMainRuntimeCompartmentsSystem");
     c("MEMORY_JS_COMPARTMENTS_USER", "JSMainRuntimeCompartmentsUser");
     b("MEMORY_IMAGES_CONTENT_USED_UNCOMPRESSED", "imagesContentUsedUncompressed");
     b("MEMORY_STORAGE_SQLITE", "storageSQLite");
     cc("LOW_MEMORY_EVENTS_VIRTUAL", "lowMemoryEventsVirtual");
     cc("LOW_MEMORY_EVENTS_PHYSICAL", "lowMemoryEventsPhysical");
--- a/xpcom/base/nsIMemoryReporter.idl
+++ b/xpcom/base/nsIMemoryReporter.idl
@@ -340,18 +340,18 @@ interface nsIMemoryReporterManager : nsI
    * together.
    *
    * |residentPeak| (UNITS_BYTES)  The peak resident size.
    *
    * |residentUnique| (UNITS_BYTES)  The unique set size (a.k.a. USS).
    *
    * |heapAllocated| (UNITS_BYTES)  Memory mapped by the heap allocator.
    *
-   * |heapOverheadRatio| (UNITS_PERCENTAGE)  In the heap allocator, this is the
-   * ratio of committed, unused bytes to allocated bytes.  Like all
+   * |heapOverheadFraction| (UNITS_PERCENTAGE)  In the heap allocator, this is
+   * the fraction of committed heap bytes that are overhead. Like all
    * UNITS_PERCENTAGE measurements, its amount is multiplied by 100x so it can
    * be represented by an int64_t.
    *
    * |JSMainRuntimeGCHeap| (UNITS_BYTES)  Size of the main JS runtime's GC
    * heap.
    *
    * |JSMainRuntimeTemporaryPeak| (UNITS_BYTES)  Peak size of the transient
    * storage in the main JSRuntime.
@@ -376,17 +376,17 @@ interface nsIMemoryReporterManager : nsI
   readonly attribute int64_t vsize;
   readonly attribute int64_t vsizeMaxContiguous;
   readonly attribute int64_t resident;
   readonly attribute int64_t residentFast;
   readonly attribute int64_t residentPeak;
   readonly attribute int64_t residentUnique;
 
   readonly attribute int64_t heapAllocated;
-  readonly attribute int64_t heapOverheadRatio;
+  readonly attribute int64_t heapOverheadFraction;
 
   readonly attribute int64_t JSMainRuntimeGCHeap;
   readonly attribute int64_t JSMainRuntimeTemporaryPeak;
   readonly attribute int64_t JSMainRuntimeCompartmentsSystem;
   readonly attribute int64_t JSMainRuntimeCompartmentsUser;
 
   readonly attribute int64_t imagesContentUsedUncompressed;
 
--- a/xpcom/base/nsMemoryReporterManager.cpp
+++ b/xpcom/base/nsMemoryReporterManager.cpp
@@ -1270,23 +1270,31 @@ NS_IMPL_ISUPPORTS(PageFaultsHardReporter
 /**
  ** memory reporter implementation for jemalloc and OSX malloc,
  ** to obtain info on total memory in use (that we know about,
  ** at least -- on OSX, there are sometimes other zones in use).
  **/
 
 #ifdef HAVE_JEMALLOC_STATS
 
-// This has UNITS_PERCENTAGE, so it is multiplied by 100.
-static int64_t
-HeapOverheadRatio(jemalloc_stats_t* aStats)
+static size_t
+HeapOverhead(jemalloc_stats_t* aStats)
 {
-  return (int64_t)10000 *
-    (aStats->waste + aStats->bookkeeping + aStats->page_cache) /
-    ((double)aStats->allocated);
+  return aStats->waste + aStats->bookkeeping +
+         aStats->page_cache + aStats->bin_unused;
+}
+
+// This has UNITS_PERCENTAGE, so it is multiplied by 100x *again* on top of the
+// 100x for the percentage.
+static int64_t
+HeapOverheadFraction(jemalloc_stats_t* aStats)
+{
+  size_t heapOverhead = HeapOverhead(aStats);
+  size_t heapCommitted = aStats->allocated + heapOverhead;
+  return int64_t(10000 * (heapOverhead / (double)heapCommitted));
 }
 
 class JemallocHeapReporter final : public nsIMemoryReporter
 {
   ~JemallocHeapReporter() {}
 
 public:
   NS_DECL_ISUPPORTS
@@ -1295,43 +1303,47 @@ public:
                            nsISupports* aData, bool aAnonymize) override
   {
     jemalloc_stats_t stats;
     jemalloc_stats(&stats);
 
     nsresult rv;
 
     rv = MOZ_COLLECT_REPORT(
-      "heap-allocated", KIND_OTHER, UNITS_BYTES, stats.allocated,
+      "heap-committed/allocated", KIND_OTHER, UNITS_BYTES, stats.allocated,
 "Memory mapped by the heap allocator that is currently allocated to the "
 "application.  This may exceed the amount of memory requested by the "
 "application because the allocator regularly rounds up request sizes. (The "
 "exact amount requested is not recorded.)");
     NS_ENSURE_SUCCESS(rv, rv);
 
+    rv = MOZ_COLLECT_REPORT(
+      "heap-allocated", KIND_OTHER, UNITS_BYTES, stats.allocated,
+"The same as 'heap-committed/allocated'.");
+    NS_ENSURE_SUCCESS(rv, rv);
+
     // We mark this and the other heap-overhead reporters as KIND_NONHEAP
     // because KIND_HEAP memory means "counted in heap-allocated", which
     // this is not.
     rv = MOZ_COLLECT_REPORT(
       "explicit/heap-overhead/bin-unused", KIND_NONHEAP, UNITS_BYTES,
       stats.bin_unused,
-"Bytes reserved for bins of fixed-size allocations which do not correspond to "
-"an active allocation.");
+"Unused bytes due to fragmentation in the bins used for 'small' (<= 2 KiB) "
+"allocations. These bytes will be used if additional allocations occur.");
     NS_ENSURE_SUCCESS(rv, rv);
 
-    rv = MOZ_COLLECT_REPORT(
-      "explicit/heap-overhead/waste", KIND_NONHEAP, UNITS_BYTES,
-      stats.waste,
+    if (stats.waste > 0) {
+      rv = MOZ_COLLECT_REPORT(
+        "explicit/heap-overhead/waste", KIND_NONHEAP, UNITS_BYTES,
+        stats.waste,
 "Committed bytes which do not correspond to an active allocation and which the "
-"allocator is not intentionally keeping alive (i.e., not 'heap-bookkeeping' or "
-"'heap-page-cache' or 'heap-bin-unused').  Although the allocator will waste "
-"some space under any circumstances, a large value here may indicate that the "
-"heap is highly fragmented, or that allocator is performing poorly for some "
-"other reason.");
-    NS_ENSURE_SUCCESS(rv, rv);
+"allocator is not intentionally keeping alive (i.e., not "
+"'explicit/heap-overhead/{bookkeeping,page-cache,bin-unused}').");
+      NS_ENSURE_SUCCESS(rv, rv);
+    }
 
     rv = MOZ_COLLECT_REPORT(
       "explicit/heap-overhead/bookkeeping", KIND_NONHEAP, UNITS_BYTES,
       stats.bookkeeping,
 "Committed bytes which the heap allocator uses for internal data structures.");
     NS_ENSURE_SUCCESS(rv, rv);
 
     rv = MOZ_COLLECT_REPORT(
@@ -1339,48 +1351,32 @@ public:
       stats.page_cache,
 "Memory which the allocator could return to the operating system, but hasn't. "
 "The allocator keeps this memory around as an optimization, so it doesn't "
 "have to ask the OS the next time it needs to fulfill a request. This value "
 "is typically not larger than a few megabytes.");
     NS_ENSURE_SUCCESS(rv, rv);
 
     rv = MOZ_COLLECT_REPORT(
-      "heap-committed", KIND_OTHER, UNITS_BYTES,
-      stats.allocated + stats.waste + stats.bookkeeping + stats.page_cache,
-"Memory mapped by the heap allocator that is committed, i.e. in physical "
-"memory or paged to disk.  This value corresponds to 'heap-allocated' + "
-"'heap-waste' + 'heap-bookkeeping' + 'heap-page-cache', but because "
-"these values are read at different times, the result probably won't match "
-"exactly.");
-    NS_ENSURE_SUCCESS(rv, rv);
-
-    rv = MOZ_COLLECT_REPORT(
-      "heap-overhead-ratio", KIND_OTHER, UNITS_PERCENTAGE,
-      HeapOverheadRatio(&stats),
-"Ratio of committed, unused bytes to allocated bytes; i.e., "
-"'heap-overhead' / 'heap-allocated'.  This measures the overhead of "
-"the heap allocator relative to amount of memory allocated.");
+      "heap-committed/overhead", KIND_OTHER, UNITS_BYTES,
+      HeapOverhead(&stats),
+"The sum of 'explicit/heap-overhead/*'.");
     NS_ENSURE_SUCCESS(rv, rv);
 
     rv = MOZ_COLLECT_REPORT(
       "heap-mapped", KIND_OTHER, UNITS_BYTES, stats.mapped,
-      "Amount of memory currently mapped.");
+"Amount of memory currently mapped. Includes memory that is uncommitted, i.e. "
+"neither in physical memory nor paged to disk.");
     NS_ENSURE_SUCCESS(rv, rv);
 
     rv = MOZ_COLLECT_REPORT(
       "heap-chunksize", KIND_OTHER, UNITS_BYTES, stats.chunksize,
       "Size of chunks.");
     NS_ENSURE_SUCCESS(rv, rv);
 
-    rv = MOZ_COLLECT_REPORT(
-      "heap-chunks", KIND_OTHER, UNITS_COUNT, (stats.mapped / stats.chunksize),
-      "Number of chunks currently mapped.");
-    NS_ENSURE_SUCCESS(rv, rv);
-
     return NS_OK;
   }
 };
 NS_IMPL_ISUPPORTS(JemallocHeapReporter, nsIMemoryReporter)
 
 #endif  // HAVE_JEMALLOC_STATS
 
 // Why is this here?  At first glance, you'd think it could be defined and
@@ -2383,22 +2379,22 @@ nsMemoryReporterManager::GetHeapAllocate
 #else
   *aAmount = 0;
   return NS_ERROR_NOT_AVAILABLE;
 #endif
 }
 
 // This has UNITS_PERCENTAGE, so it is multiplied by 100x.
 NS_IMETHODIMP
-nsMemoryReporterManager::GetHeapOverheadRatio(int64_t* aAmount)
+nsMemoryReporterManager::GetHeapOverheadFraction(int64_t* aAmount)
 {
 #ifdef HAVE_JEMALLOC_STATS
   jemalloc_stats_t stats;
   jemalloc_stats(&stats);
-  *aAmount = HeapOverheadRatio(&stats);
+  *aAmount = HeapOverheadFraction(&stats);
   return NS_OK;
 #else
   *aAmount = 0;
   return NS_ERROR_NOT_AVAILABLE;
 #endif
 }
 
 static nsresult