Bug 660731 - Add GetExplicit and GetResident methods to NSIMemoryReporterManager, attempt 2. r=khuey, sr=bz.
☠☠ backed out by 77144b649993 ☠ ☠
authorNicholas Nethercote <nnethercote@mozilla.com>
Tue, 05 Jul 2011 15:34:41 +1000
changeset 72316 30f74655c985e67c020fb5db3bdd838da1e03700
parent 72315 cceff8f75d6a4e0ab44f272bb6aacbc1e31efd35
child 72317 a733e437f95aac6980f5307eb5fc8782066217e9
push idunknown
push userunknown
push dateunknown
reviewerskhuey, bz
bugs660731
milestone7.0a1
Bug 660731 - Add GetExplicit and GetResident methods to NSIMemoryReporterManager, attempt 2. r=khuey, sr=bz.
dom/ipc/ContentChild.cpp
toolkit/components/aboutmemory/tests/chrome/test_aboutmemory.xul
toolkit/components/telemetry/TelemetryHistograms.h
toolkit/components/telemetry/TelemetryPing.js
xpcom/base/nsIMemoryReporter.idl
xpcom/base/nsMemoryReporterManager.cpp
--- a/dom/ipc/ContentChild.cpp
+++ b/dom/ipc/ContentChild.cpp
@@ -372,23 +372,23 @@ ContentChild::RecvPMemoryReportRequestCo
       MemoryReport memreport(process, path, kind, units, amount, desc);
       reports.AppendElement(memreport);
     }
 
     // Then do the memory multi-reporters, by calling CollectReports on each
     // one, whereupon the callback will turn each measurement into a
     // MemoryReport.
     mgr->EnumerateMultiReporters(getter_AddRefs(e));
-    MemoryReportsWrapper wrappedReports(&reports);
-    MemoryReportCallback cb(process);
+    nsRefPtr<MemoryReportsWrapper> wrappedReports =
+        new MemoryReportsWrapper(&reports);
+    nsRefPtr<MemoryReportCallback> cb = new MemoryReportCallback(process);
     while (NS_SUCCEEDED(e->HasMoreElements(&more)) && more) {
       nsCOMPtr<nsIMemoryMultiReporter> r;
       e->GetNext(getter_AddRefs(r));
-
-      r->CollectReports(&cb, &wrappedReports);
+      r->CollectReports(cb, wrappedReports);
     }
 
     child->Send__delete__(child, reports);
     return true;
 }
 
 bool
 ContentChild::DeallocPMemoryReportRequest(PMemoryReportRequestChild* actor)
--- a/toolkit/components/aboutmemory/tests/chrome/test_aboutmemory.xul
+++ b/toolkit/components/aboutmemory/tests/chrome/test_aboutmemory.xul
@@ -72,27 +72,31 @@
   var fakeReporters = [
     f("", "heap-used",          OTHER,  500 * MB),
     f("", "heap-unused",        OTHER,  100 * MB),
     f("", "explicit/a",         HEAP,   222 * MB),
     f("", "explicit/b/a",       HEAP,    85 * MB),
     f("", "explicit/b/b",       HEAP,    75 * MB),
     f("", "explicit/b/c/a",     HEAP,    70 * MB),
     f("", "explicit/b/c/b",     HEAP,     2 * MB), // omitted
+    f("", "explicit/c",         MAPPED, 100 * MB),
+    f("", "explicit/c/d",       MAPPED,  13 * MB), // subsumed by parent
     f("", "explicit/g",         HEAP,     1 * MB), // internal, dup: merge
     f("", "explicit/g/a",       HEAP,     6 * MB),
     f("", "explicit/g/b",       HEAP,     5 * MB),
     f("", "other1",             OTHER,  111 * MB),
     f2("", "other4",            OTHER,  COUNT, 888),
     f2("", "unknown-unit",      OTHER,  /*bogus unit*/999, 999)
   ];
   var fakeMultiReporters = [
      { collectReports: function(cbObj, closure) {
           function f(p, k, u, a) { cbObj.callback("", p, k, u, a, "(desc)", closure); }
-          f("explicit/c",       MAPPED, BYTES, 123 * MB);
+          f("explicit/c/d",     MAPPED, BYTES,  10 * MB), // dup, subsumed by parent
+          f("explicit/cc",      MAPPED, BYTES,  13 * MB);
+          f("explicit/cc",      MAPPED, BYTES,  10 * MB); // dup
           f("explicit/d",       MAPPED, BYTES, 499 * KB); // omitted
           f("explicit/e",       MAPPED, BYTES, 100 * KB); // omitted
           f("explicit/f/g/h/i", HEAP,   BYTES,  20 * MB);
        }
      },
      { collectReports: function(cbObj, closure) {
           function f(p, k, u, a) { cbObj.callback("", p, k, u, a, "(desc)", closure); }
           f("explicit/g",       HEAP,   BYTES,  14 * MB); // internal
@@ -103,16 +107,27 @@
   ];
   for (var i = 0; i < fakeReporters.length; i++) {
     mgr.registerReporter(fakeReporters[i]);
   }
   for (var i = 0; i < fakeMultiReporters.length; i++) {
     mgr.registerMultiReporter(fakeMultiReporters[i]);
   }
 
+  // mgr.explicit sums "heap-used" and all the appropriate MAPPED ones:
+  // - "explicit/c", "explicit/cc" x 2, "explicit/d", "explicit/e"
+  // - but *not* "explicit/c/d" x 2
+  // Check explicit now before we add the fake reporters for the fake 2nd
+  // and subsequent processes.
+  is(mgr.explicit, 500*MB + (100 + 13 + 10)*MB + 599*KB, "mgr.explicit");
+ 
+  // Access mgr.resident just to make sure it doesn't crash.  We can't check
+  // its actual value because it's non-deterministic.
+  dummy = mgr.resident;
+
   var fakeReporters2 = [
     f("2nd", "heap-used",       OTHER, 1000 * MB),
     f("2nd", "heap-unused",     OTHER,  100 * MB),
     f("2nd", "explicit/a/b/c",  HEAP,   498 * MB),
     f("2nd", "explicit/a/b/c",  HEAP,     1 * MB), // dup: merge
     f("2nd", "explicit/flip\\the\\backslashes",
                                 HEAP,   200 * MB),
     f("2nd", "explicit/compartment(this-will-be-truncated-in-non-verbose-mode-abcdefghijklmnopqrstuvwxyz)",
@@ -152,17 +167,20 @@ Explicit Allocations\n\
 623.58 MB (100.0%) -- explicit\n\
 ├──232.00 MB (37.20%) -- b\n\
 │  ├───85.00 MB (13.63%) -- a\n\
 │  ├───75.00 MB (12.03%) -- b\n\
 │  └───72.00 MB (11.55%) -- c\n\
 │      ├──70.00 MB (11.23%) -- a\n\
 │      └───2.00 MB (00.32%) -- (1 omitted)\n\
 ├──222.00 MB (35.60%) -- a\n\
-├──123.00 MB (19.72%) -- c\n\
+├──100.00 MB (16.04%) -- c\n\
+│  ├───77.00 MB (12.35%) -- other\n\
+│  └───23.00 MB (03.69%) -- d\n\
+├───23.00 MB (03.69%) -- cc\n\
 ├───20.00 MB (03.21%) -- f\n\
 │   └──20.00 MB (03.21%) -- g\n\
 │      └──20.00 MB (03.21%) -- h\n\
 │         └──20.00 MB (03.21%) -- i\n\
 ├───15.00 MB (02.41%) -- g\n\
 │   ├───6.00 MB (00.96%) -- a\n\
 │   ├───5.00 MB (00.80%) -- b\n\
 │   └───4.00 MB (00.64%) -- other\n\
@@ -219,17 +237,20 @@ Explicit Allocations\n\
 653,876,224 B (100.0%) -- explicit\n\
 ├──243,269,632 B (37.20%) -- b\n\
 │  ├───89,128,960 B (13.63%) -- a\n\
 │  ├───78,643,200 B (12.03%) -- b\n\
 │  └───75,497,472 B (11.55%) -- c\n\
 │      ├──73,400,320 B (11.23%) -- a\n\
 │      └───2,097,152 B (00.32%) -- b\n\
 ├──232,783,872 B (35.60%) -- a\n\
-├──128,974,848 B (19.72%) -- c\n\
+├──104,857,600 B (16.04%) -- c\n\
+│  ├───80,740,352 B (12.35%) -- other\n\
+│  └───24,117,248 B (03.69%) -- d\n\
+├───24,117,248 B (03.69%) -- cc\n\
 ├───20,971,520 B (03.21%) -- f\n\
 │   └──20,971,520 B (03.21%) -- g\n\
 │      └──20,971,520 B (03.21%) -- h\n\
 │         └──20,971,520 B (03.21%) -- i\n\
 ├───15,728,640 B (02.41%) -- g\n\
 │   ├───6,291,456 B (00.96%) -- a\n\
 │   ├───5,242,880 B (00.80%) -- b\n\
 │   └───4,194,304 B (00.64%) -- other\n\
--- a/toolkit/components/telemetry/TelemetryHistograms.h
+++ b/toolkit/components/telemetry/TelemetryHistograms.h
@@ -50,18 +50,17 @@ HISTOGRAM(CYCLE_COLLECTOR_VISITED_GCED, 
 HISTOGRAM(CYCLE_COLLECTOR_COLLECTED, 1, 100000, 50, EXPONENTIAL, "Number of objects collected by the cycle collector")
 HISTOGRAM(TELEMETRY_PING, 1, 3000, 10, EXPONENTIAL, "Time taken to submit telemetry info (ms)")
 HISTOGRAM(TELEMETRY_SUCCESS, 0, 1, 2, BOOLEAN,  "Successful telemetry submission")
 HISTOGRAM(MEMORY_JS_GC_HEAP, 1024, 512 * 1024, 10, EXPONENTIAL, "Memory used by the garbage-collected JavaScript heap (KB)")
 HISTOGRAM(MEMORY_RESIDENT, 32 * 1024, 1024 * 1024, 10, EXPONENTIAL, "Resident memory size (KB)")
 HISTOGRAM(MEMORY_LAYOUT_ALL, 1024, 64 * 1024, 10, EXPONENTIAL, "Memory used by layout (KB)")
 HISTOGRAM(MEMORY_IMAGES_CONTENT_USED_UNCOMPRESSED, 1024, 1024 * 1024, 10, EXPONENTIAL, "Memory used for uncompressed, in-use content images (KB)")
 HISTOGRAM(MEMORY_HEAP_USED, 1024, 1024 * 1024, 10, EXPONENTIAL, "Heap memory used (KB)")
-// XXX: bug 660731 will enable this
-//HISTOGRAM(MEMORY_EXPLICIT, 1024, 1024 * 1024, 10, EXPONENTIAL, "Explicit memory allocations (KB)")
+HISTOGRAM(MEMORY_EXPLICIT, 1024, 1024 * 1024, 10, EXPONENTIAL, "Explicit memory allocations (KB)")
 #if defined(XP_WIN)
 HISTOGRAM(EARLY_GLUESTARTUP_READ_OPS, 1, 100, 12, LINEAR, "ProcessIoCounters.ReadOperationCount before glue startup")
 HISTOGRAM(EARLY_GLUESTARTUP_READ_TRANSFER, 1, 50 * 1024, 12, EXPONENTIAL, "ProcessIoCounters.ReadTransferCount before glue startup (KB)")
 HISTOGRAM(GLUESTARTUP_READ_OPS, 1, 100, 12, LINEAR, "ProcessIoCounters.ReadOperationCount after glue startup")
 HISTOGRAM(GLUESTARTUP_READ_TRANSFER, 1, 50 * 1024, 12, EXPONENTIAL, "ProcessIoCounters.ReadTransferCount after glue startup (KB)")
 #elif defined(XP_UNIX)
 HISTOGRAM(EARLY_GLUESTARTUP_HARD_FAULTS, 1, 100, 12, LINEAR, "Hard faults count before glue startup")
 HISTOGRAM(GLUESTARTUP_HARD_FAULTS, 1, 500, 12, EXPONENTIAL, "Hard faults count after glue startup")
--- a/toolkit/components/telemetry/TelemetryPing.js
+++ b/toolkit/components/telemetry/TelemetryPing.js
@@ -246,19 +246,21 @@ TelemetryPing.prototype = {
         this._prevValues[mr.path] = curVal;
       }
       else {
         NS_ASSERT(false, "Can't handle memory reporter with units " + mr.units);
         continue;
       }
       this.addValue(mr.path, id, val);
     }
-    // XXX: bug 660731 will enable this
     // "explicit" is found differently.
-    //this.addValue("explicit", "MEMORY_EXPLICIT", Math.floor(mgr.explicit / 1024));
+    let explicit = mgr.explicit;    // Get it only once, it's reasonably expensive
+    if (explicit != -1) {
+      this.addValue("explicit", "MEMORY_EXPLICIT", Math.floor(explicit / 1024));
+    }
   },
   
   /**
    * Send data to the server. Record success/send-time in histograms
    */
   send: function send(reason, server) {
     // populate histograms one last time
     this.gatherMemory();
--- a/xpcom/base/nsIMemoryReporter.idl
+++ b/xpcom/base/nsIMemoryReporter.idl
@@ -136,17 +136,18 @@ interface nsIMemoryReporter : nsISupport
   const PRInt32 UNITS_COUNT = 1;
 
   /*
    * The units on the reporter's amount.  See UNITS_* above.
    */
   readonly attribute PRInt32 units;
 
   /*
-   * The numeric value reported by this memory reporter.
+   * The numeric value reported by this memory reporter.  -1 means "unknown",
+   * ie. something went wrong when getting the amount.
    */
   readonly attribute PRInt64 amount;
 
   /*
    * A human-readable description of this memory usage report.
    */
   readonly attribute string description;
 };
@@ -173,17 +174,17 @@ interface nsIMemoryMultiReporterCallback
  */
 [scriptable, uuid(eae277ad-b67d-4389-95f4-03fa11c09d06)]
 interface nsIMemoryMultiReporter : nsISupports
 {
   void collectReports(in nsIMemoryMultiReporterCallback callback,
                       in nsISupports closure);
 };
 
-[scriptable, uuid(80a93b4c-6fff-4acd-8598-3891074a30ab)]
+[scriptable, uuid(84ba9c85-3372-4423-b7ab-74708b9269a6)]
 interface nsIMemoryReporterManager : nsISupports
 {
   /*
    * Return an enumerator of nsIMemoryReporters that are currently registered.
    */
   nsISimpleEnumerator enumerateReporters ();
 
   /*
@@ -216,16 +217,34 @@ interface nsIMemoryReporterManager : nsI
    * Unregister the given memory multi-reporter.
    */
   void unregisterMultiReporter (in nsIMemoryMultiReporter reporter);
 
   /*
    * Initialize.
    */
   void init ();
+
+  /*
+   * Get the resident size (aka. RSS, physical memory used).  This reporter
+   * is special-cased because it's interesting, is available on all
+   * platforms, and returns a meaningful result on all common platforms.
+   * -1 means unknown.
+   */
+  readonly attribute PRInt64 resident;
+
+  /*
+   * Get the total size of explicit memory allocations, both at the OS-level
+   * (eg. via mmap, VirtualAlloc) and at the heap level (eg. via malloc,
+   * calloc, operator new).  (Nb: it covers all heap allocations, but will
+   * miss any OS-level ones not covered by memory reporters.)  This reporter
+   * is special-cased because it's interesting, and is moderately difficult
+   * to compute in JS.  -1 means unknown.
+   */
+  readonly attribute PRInt64 explicit;
 };
 
 %{C++
 
 /*
  * Note that this defaults 'process' to "", which is usually what's desired.
  */
 #define NS_MEMORY_REPORTER_IMPLEMENT(_classname, _path, _kind, _units, _usageFunction, _desc) \
--- a/xpcom/base/nsMemoryReporterManager.cpp
+++ b/xpcom/base/nsMemoryReporterManager.cpp
@@ -31,20 +31,22 @@
  * use your version of this file under the terms of the MPL, indicate your
  * decision by deleting the provisions above and replace them with the notice
  * and other provisions required by the GPL or the LGPL. If you do not delete
  * the provisions above, a recipient may use your version of this file under
  * the terms of any one of the MPL, the GPL or the LGPL.
  *
  * ***** END LICENSE BLOCK ***** */
 
+#include "nsAutoPtr.h"
 #include "nsCOMPtr.h"
 #include "nsServiceManagerUtils.h"
 #include "nsMemoryReporterManager.h"
 #include "nsArrayEnumerator.h"
+#include "nsISimpleEnumerator.h"
 
 #if defined(XP_LINUX) || defined(XP_MACOSX)
 
 #include <sys/time.h>
 #include <sys/resource.h>
 
 static PRInt64 GetHardPageFaults()
 {
@@ -502,16 +504,174 @@ nsMemoryReporterManager::UnregisterMulti
 {
     mozilla::MutexAutoLock autoLock(mMutex);
     if (!mMultiReporters.RemoveObject(reporter))
         return NS_ERROR_FAILURE;
 
     return NS_OK;
 }
 
+NS_IMETHODIMP
+nsMemoryReporterManager::GetResident(PRInt64 *aResident)
+{
+    *aResident = ::GetResident();
+    return NS_OK;
+}
+
+struct MemoryReport {
+    MemoryReport(const nsACString &path, PRInt64 amount) 
+    : path(path), amount(amount)
+    {
+        MOZ_COUNT_CTOR(MemoryReport);
+    }
+    ~MemoryReport() 
+    {
+        MOZ_COUNT_DTOR(MemoryReport);
+    }
+    const nsCString path;
+    PRInt64 amount;
+};
+
+// This is just a wrapper for InfallibleTArray<MemoryReport> that implements
+// nsISupports, so it can be passed to nsIMemoryMultiReporter::CollectReports.
+class MemoryReportsWrapper : public nsISupports {
+public:
+    NS_DECL_ISUPPORTS
+    MemoryReportsWrapper(InfallibleTArray<MemoryReport> *r) : mReports(r) { }
+    InfallibleTArray<MemoryReport> *mReports;
+};
+NS_IMPL_ISUPPORTS0(MemoryReportsWrapper)
+
+class MemoryReportCallback : public nsIMemoryMultiReporterCallback
+{
+public:
+    NS_DECL_ISUPPORTS
+
+    NS_IMETHOD Callback(const nsACString &aProcess, const nsACString &aPath,
+                        PRInt32 aKind, PRInt32 aUnits, PRInt64 aAmount,
+                        const nsACString &aDescription,
+                        nsISupports *aWrappedMRs)
+    {
+        if (aKind == nsIMemoryReporter::KIND_MAPPED && aAmount != PRInt64(-1)) {
+            MemoryReportsWrapper *wrappedMRs =
+                static_cast<MemoryReportsWrapper *>(aWrappedMRs);
+            MemoryReport mr(aPath, aAmount);
+            wrappedMRs->mReports->AppendElement(mr);
+        }
+        return NS_OK;
+    }
+};
+NS_IMPL_ISUPPORTS1(
+  MemoryReportCallback
+, nsIMemoryMultiReporterCallback
+)
+
+// Is path1 a prefix, and thus a parent, of path2?  Eg. "a/b" is a parent of
+// "a/b/c", but "a/bb" is not.
+static bool
+isParent(const nsACString &path1, const nsACString &path2)
+{
+    if (path1.Length() >= path2.Length())
+        return false;
+
+    const nsACString& subStr = Substring(path2, 0, path1.Length());
+    return subStr.Equals(path1) && path2[path1.Length()] == '/';
+}
+
+NS_IMETHODIMP
+nsMemoryReporterManager::GetExplicit(PRInt64 *aExplicit)
+{
+    InfallibleTArray<MemoryReport> mapped;
+    PRInt64 heapUsed = PRInt64(-1);
+
+    // Get "heap-used" and all the KIND_MAPPED measurements from vanilla
+    // reporters.
+    nsCOMPtr<nsISimpleEnumerator> e;
+    EnumerateReporters(getter_AddRefs(e));
+
+    PRBool more;
+    while (NS_SUCCEEDED(e->HasMoreElements(&more)) && more) {
+        nsCOMPtr<nsIMemoryReporter> r;
+        e->GetNext(getter_AddRefs(r));
+
+        PRInt32 kind;
+        nsresult rv = r->GetKind(&kind);
+        NS_ENSURE_SUCCESS(rv, rv);
+
+        if (kind == nsIMemoryReporter::KIND_MAPPED) {
+            nsCString path;
+            rv = r->GetPath(getter_Copies(path));
+            NS_ENSURE_SUCCESS(rv, rv);
+
+            PRInt64 amount;
+            rv = r->GetAmount(&amount);
+            NS_ENSURE_SUCCESS(rv, rv);
+
+            // Just skip any MAPPED reporters that fail, because "heap-used" is
+            // the most important one.
+            if (amount != PRInt64(-1)) {
+                MemoryReport mr(path, amount);
+                mapped.AppendElement(mr);
+            }
+        } else {
+            nsCString path;
+            rv = r->GetPath(getter_Copies(path));
+            NS_ENSURE_SUCCESS(rv, rv);
+
+            if (path.Equals("heap-used")) {
+                rv = r->GetAmount(&heapUsed);
+                NS_ENSURE_SUCCESS(rv, rv);
+                // If "heap-used" fails, we give up, because the result would be
+                // horribly inaccurate.
+                if (heapUsed == PRInt64(-1)) {
+                    *aExplicit = PRInt64(-1);
+                    return NS_OK;
+                }
+            }
+        }
+    }
+
+    // Get KIND_MAPPED measurements from multi-reporters, too.
+    nsCOMPtr<nsISimpleEnumerator> e2;
+    EnumerateMultiReporters(getter_AddRefs(e2));
+    nsRefPtr<MemoryReportsWrapper> wrappedMRs =
+        new MemoryReportsWrapper(&mapped);
+    nsRefPtr<MemoryReportCallback> cb = new MemoryReportCallback();
+
+    while (NS_SUCCEEDED(e2->HasMoreElements(&more)) && more) {
+      nsCOMPtr<nsIMemoryMultiReporter> r;
+      e2->GetNext(getter_AddRefs(r));
+      r->CollectReports(cb, wrappedMRs);
+    }
+
+    // Ignore (by zeroing its amount) any reporter that is a child of another
+    // reporter.  Eg. if we have "explicit/a" and "explicit/a/b", zero the
+    // latter.  This is quadratic in the number of MAPPED reporters, but there
+    // shouldn't be many.
+    for (PRUint32 i = 0; i < mapped.Length(); i++) {
+        const nsCString &iPath = mapped[i].path;
+        for (PRUint32 j = i + 1; j < mapped.Length(); j++) {
+            const nsCString &jPath = mapped[j].path;
+            if (isParent(iPath, jPath)) {
+                mapped[j].amount = 0;
+            } else if (isParent(jPath, iPath)) {
+                mapped[i].amount = 0;
+            }
+        }
+    }
+
+    // Sum all the mapped reporters and heapUsed.
+    *aExplicit = heapUsed;
+    for (PRUint32 i = 0; i < mapped.Length(); i++) {
+        *aExplicit += mapped[i].amount;
+    }
+
+    return NS_OK;
+}
+
 NS_IMPL_ISUPPORTS1(nsMemoryReporter, nsIMemoryReporter)
 
 nsMemoryReporter::nsMemoryReporter(nsCString& process,
                                    nsCString& path,
                                    PRInt32 kind,
                                    PRInt32 units,
                                    PRInt64 amount,
                                    nsCString& desc)