Bug 660731 - Add GetExplicit and GetResident methods to NSIMemoryReporterManager, attempt 3. r=khuey, sr=bz.

dom/ipc/ContentChild.cpp

@@ -377,13 +377,13 @@ ContentChild::RecvPMemoryReportRequestConstructor(PMemoryReportRequestChild* chi
     // one, whereupon the callback will turn each measurement into a
     // MemoryReport.
     mgr->EnumerateMultiReporters(getter_AddRefs(e));
-    MemoryReportsWrapper wrappedReports(&reports);
+    nsRefPtr<MemoryReportsWrapper> wrappedReports =
-    MemoryReportCallback cb(process);
+        new MemoryReportsWrapper(&reports);
+    nsRefPtr<MemoryReportCallback> cb = new MemoryReportCallback(process);
     while (NS_SUCCEEDED(e->HasMoreElements(&more)) && more) {
       nsCOMPtr<nsIMemoryMultiReporter> r;
       e->GetNext(getter_AddRefs(r));
-
+      r->CollectReports(cb, wrappedReports);
-      r->CollectReports(&cb, &wrappedReports);
     }
 
     child->Send__delete__(child, reports);

toolkit/components/aboutmemory/tests/chrome/test_aboutmemory.xul

@@ -77,6 +77,8 @@
     f("", "explicit/b/b",       HEAP,    75 * MB),
     f("", "explicit/b/c/a",     HEAP,    70 * MB),
     f("", "explicit/b/c/b",     HEAP,     2 * MB), // omitted
+    f("", "explicit/c",         MAPPED, 100 * MB),
+    f("", "explicit/c/d",       MAPPED,  13 * MB), // subsumed by parent
     f("", "explicit/g",         HEAP,     1 * MB), // internal, dup: merge
     f("", "explicit/g/a",       HEAP,     6 * MB),
     f("", "explicit/g/b",       HEAP,     5 * MB),
@@ -87,7 +89,9 @@
   var fakeMultiReporters = [
      { collectReports: function(cbObj, closure) {
           function f(p, k, u, a) { cbObj.callback("", p, k, u, a, "(desc)", closure); }
-          f("explicit/c",       MAPPED, BYTES, 123 * MB);
+          f("explicit/c/d",     MAPPED, BYTES,  10 * MB), // dup, subsumed by parent
+          f("explicit/cc",      MAPPED, BYTES,  13 * MB);
+          f("explicit/cc",      MAPPED, BYTES,  10 * MB); // dup
           f("explicit/d",       MAPPED, BYTES, 499 * KB); // omitted
           f("explicit/e",       MAPPED, BYTES, 100 * KB); // omitted
           f("explicit/f/g/h/i", HEAP,   BYTES,  20 * MB);
@@ -108,6 +112,17 @@
     mgr.registerMultiReporter(fakeMultiReporters[i]);
   }
 
+  // mgr.explicit sums "heap-used" and all the appropriate MAPPED ones:
+  // - "explicit/c", "explicit/cc" x 2, "explicit/d", "explicit/e"
+  // - but *not* "explicit/c/d" x 2
+  // Check explicit now before we add the fake reporters for the fake 2nd
+  // and subsequent processes.
+  is(mgr.explicit, 500*MB + (100 + 13 + 10)*MB + 599*KB, "mgr.explicit");
+ 
+  // Access mgr.resident just to make sure it doesn't crash.  We can't check
+  // its actual value because it's non-deterministic.
+  dummy = mgr.resident;
+
   var fakeReporters2 = [
     f("2nd", "heap-used",       OTHER, 1000 * MB),
     f("2nd", "heap-unused",     OTHER,  100 * MB),
@@ -157,7 +172,10 @@ Explicit Allocations\n\
 │      ├──70.00 MB (11.23%) -- a\n\
 │      └───2.00 MB (00.32%) -- (1 omitted)\n\
 ├──222.00 MB (35.60%) -- a\n\
-├──123.00 MB (19.72%) -- c\n\
+├──100.00 MB (16.04%) -- c\n\
+│  ├───77.00 MB (12.35%) -- other\n\
+│  └───23.00 MB (03.69%) -- d\n\
+├───23.00 MB (03.69%) -- cc\n\
 ├───20.00 MB (03.21%) -- f\n\
 │   └──20.00 MB (03.21%) -- g\n\
 │      └──20.00 MB (03.21%) -- h\n\
@@ -224,7 +242,10 @@ Explicit Allocations\n\
 │      ├──73,400,320 B (11.23%) -- a\n\
 │      └───2,097,152 B (00.32%) -- b\n\
 ├──232,783,872 B (35.60%) -- a\n\
-├──128,974,848 B (19.72%) -- c\n\
+├──104,857,600 B (16.04%) -- c\n\
+│  ├───80,740,352 B (12.35%) -- other\n\
+│  └───24,117,248 B (03.69%) -- d\n\
+├───24,117,248 B (03.69%) -- cc\n\
 ├───20,971,520 B (03.21%) -- f\n\
 │   └──20,971,520 B (03.21%) -- g\n\
 │      └──20,971,520 B (03.21%) -- h\n\

toolkit/components/telemetry/TelemetryHistograms.h

@@ -55,8 +55,7 @@ HISTOGRAM(MEMORY_RESIDENT, 32 * 1024, 1024 * 1024, 10, EXPONENTIAL, "Resident me
 HISTOGRAM(MEMORY_LAYOUT_ALL, 1024, 64 * 1024, 10, EXPONENTIAL, "Memory used by layout (KB)")
 HISTOGRAM(MEMORY_IMAGES_CONTENT_USED_UNCOMPRESSED, 1024, 1024 * 1024, 10, EXPONENTIAL, "Memory used for uncompressed, in-use content images (KB)")
 HISTOGRAM(MEMORY_HEAP_USED, 1024, 1024 * 1024, 10, EXPONENTIAL, "Heap memory used (KB)")
-// XXX: bug 660731 will enable this
+HISTOGRAM(MEMORY_EXPLICIT, 1024, 1024 * 1024, 10, EXPONENTIAL, "Explicit memory allocations (KB)")
-//HISTOGRAM(MEMORY_EXPLICIT, 1024, 1024 * 1024, 10, EXPONENTIAL, "Explicit memory allocations (KB)")
 #if defined(XP_WIN)
 HISTOGRAM(EARLY_GLUESTARTUP_READ_OPS, 1, 100, 12, LINEAR, "ProcessIoCounters.ReadOperationCount before glue startup")
 HISTOGRAM(EARLY_GLUESTARTUP_READ_TRANSFER, 1, 50 * 1024, 12, EXPONENTIAL, "ProcessIoCounters.ReadTransferCount before glue startup (KB)")

toolkit/components/telemetry/TelemetryPing.js

@@ -251,9 +251,11 @@ TelemetryPing.prototype = {
       }
       this.addValue(mr.path, id, val);
     }
-    // XXX: bug 660731 will enable this
     // "explicit" is found differently.
-    //this.addValue("explicit", "MEMORY_EXPLICIT", Math.floor(mgr.explicit / 1024));
+    let explicit = mgr.explicit;    // Get it only once, it's reasonably expensive
+    if (explicit != -1) {
+      this.addValue("explicit", "MEMORY_EXPLICIT", Math.floor(explicit / 1024));
+    }
   },
   
   /**

xpcom/base/nsIMemoryReporter.idl

@@ -141,7 +141,8 @@ interface nsIMemoryReporter : nsISupports
   readonly attribute PRInt32 units;
 
   /*
-   * The numeric value reported by this memory reporter.
+   * The numeric value reported by this memory reporter.  -1 means "unknown",
+   * ie. something went wrong when getting the amount.
    */
   readonly attribute PRInt64 amount;
 
@@ -178,7 +179,7 @@ interface nsIMemoryMultiReporter : nsISupports
                       in nsISupports closure);
 };
 
-[scriptable, uuid(80a93b4c-6fff-4acd-8598-3891074a30ab)]
+[scriptable, uuid(84ba9c85-3372-4423-b7ab-74708b9269a6)]
 interface nsIMemoryReporterManager : nsISupports
 {
   /*
@@ -221,6 +222,24 @@ interface nsIMemoryReporterManager : nsISupports
    * Initialize.
    */
   void init ();
+
+  /*
+   * Get the resident size (aka. RSS, physical memory used).  This reporter
+   * is special-cased because it's interesting, is available on all
+   * platforms, and returns a meaningful result on all common platforms.
+   * -1 means unknown.
+   */
+  readonly attribute PRInt64 resident;
+
+  /*
+   * Get the total size of explicit memory allocations, both at the OS-level
+   * (eg. via mmap, VirtualAlloc) and at the heap level (eg. via malloc,
+   * calloc, operator new).  (Nb: it covers all heap allocations, but will
+   * miss any OS-level ones not covered by memory reporters.)  This reporter
+   * is special-cased because it's interesting, and is moderately difficult
+   * to compute in JS.  -1 means unknown.
+   */
+  readonly attribute PRInt64 explicit;
 };
 
 %{C++

xpcom/base/nsMemoryReporterManager.cpp

@@ -36,10 +36,12 @@
  *
  * ***** END LICENSE BLOCK ***** */
 
+#include "nsAutoPtr.h"
 #include "nsCOMPtr.h"
 #include "nsServiceManagerUtils.h"
 #include "nsMemoryReporterManager.h"
 #include "nsArrayEnumerator.h"
+#include "nsISimpleEnumerator.h"
 
 #if defined(XP_LINUX) || defined(XP_MACOSX)
 
@@ -507,6 +509,169 @@ nsMemoryReporterManager::UnregisterMultiReporter(nsIMemoryMultiReporter *reporte
     return NS_OK;
 }
 
+NS_IMETHODIMP
+nsMemoryReporterManager::GetResident(PRInt64 *aResident)
+{
+    *aResident = ::GetResident();
+    return NS_OK;
+}
+
+struct MemoryReport {
+    MemoryReport(const nsACString &path, PRInt64 amount) 
+    : path(path), amount(amount)
+    {
+        MOZ_COUNT_CTOR(MemoryReport);
+    }
+    MemoryReport(const MemoryReport& rhs)
+    : path(rhs.path), amount(rhs.amount)
+    {
+        MOZ_COUNT_CTOR(MemoryReport);
+    }
+    ~MemoryReport() 
+    {
+        MOZ_COUNT_DTOR(MemoryReport);
+    }
+    const nsCString path;
+    PRInt64 amount;
+};
+
+// This is just a wrapper for InfallibleTArray<MemoryReport> that implements
+// nsISupports, so it can be passed to nsIMemoryMultiReporter::CollectReports.
+class MemoryReportsWrapper : public nsISupports {
+public:
+    NS_DECL_ISUPPORTS
+    MemoryReportsWrapper(InfallibleTArray<MemoryReport> *r) : mReports(r) { }
+    InfallibleTArray<MemoryReport> *mReports;
+};
+NS_IMPL_ISUPPORTS0(MemoryReportsWrapper)
+
+class MemoryReportCallback : public nsIMemoryMultiReporterCallback
+{
+public:
+    NS_DECL_ISUPPORTS
+
+    NS_IMETHOD Callback(const nsACString &aProcess, const nsACString &aPath,
+                        PRInt32 aKind, PRInt32 aUnits, PRInt64 aAmount,
+                        const nsACString &aDescription,
+                        nsISupports *aWrappedMRs)
+    {
+        if (aKind == nsIMemoryReporter::KIND_MAPPED && aAmount != PRInt64(-1)) {
+            MemoryReportsWrapper *wrappedMRs =
+                static_cast<MemoryReportsWrapper *>(aWrappedMRs);
+            MemoryReport mr(aPath, aAmount);
+            wrappedMRs->mReports->AppendElement(mr);
+        }
+        return NS_OK;
+    }
+};
+NS_IMPL_ISUPPORTS1(
+  MemoryReportCallback
+, nsIMemoryMultiReporterCallback
+)
+
+// Is path1 a prefix, and thus a parent, of path2?  Eg. "a/b" is a parent of
+// "a/b/c", but "a/bb" is not.
+static bool
+isParent(const nsACString &path1, const nsACString &path2)
+{
+    if (path1.Length() >= path2.Length())
+        return false;
+
+    const nsACString& subStr = Substring(path2, 0, path1.Length());
+    return subStr.Equals(path1) && path2[path1.Length()] == '/';
+}
+
+NS_IMETHODIMP
+nsMemoryReporterManager::GetExplicit(PRInt64 *aExplicit)
+{
+    InfallibleTArray<MemoryReport> mapped;
+    PRInt64 heapUsed = PRInt64(-1);
+
+    // Get "heap-used" and all the KIND_MAPPED measurements from vanilla
+    // reporters.
+    nsCOMPtr<nsISimpleEnumerator> e;
+    EnumerateReporters(getter_AddRefs(e));
+
+    PRBool more;
+    while (NS_SUCCEEDED(e->HasMoreElements(&more)) && more) {
+        nsCOMPtr<nsIMemoryReporter> r;
+        e->GetNext(getter_AddRefs(r));
+
+        PRInt32 kind;
+        nsresult rv = r->GetKind(&kind);
+        NS_ENSURE_SUCCESS(rv, rv);
+
+        if (kind == nsIMemoryReporter::KIND_MAPPED) {
+            nsCString path;
+            rv = r->GetPath(getter_Copies(path));
+            NS_ENSURE_SUCCESS(rv, rv);
+
+            PRInt64 amount;
+            rv = r->GetAmount(&amount);
+            NS_ENSURE_SUCCESS(rv, rv);
+
+            // Just skip any MAPPED reporters that fail, because "heap-used" is
+            // the most important one.
+            if (amount != PRInt64(-1)) {
+                MemoryReport mr(path, amount);
+                mapped.AppendElement(mr);
+            }
+        } else {
+            nsCString path;
+            rv = r->GetPath(getter_Copies(path));
+            NS_ENSURE_SUCCESS(rv, rv);
+
+            if (path.Equals("heap-used")) {
+                rv = r->GetAmount(&heapUsed);
+                NS_ENSURE_SUCCESS(rv, rv);
+                // If "heap-used" fails, we give up, because the result would be
+                // horribly inaccurate.
+                if (heapUsed == PRInt64(-1)) {
+                    *aExplicit = PRInt64(-1);
+                    return NS_OK;
+                }
+            }
+        }
+    }
+
+    // Get KIND_MAPPED measurements from multi-reporters, too.
+    nsCOMPtr<nsISimpleEnumerator> e2;
+    EnumerateMultiReporters(getter_AddRefs(e2));
+    nsRefPtr<MemoryReportsWrapper> wrappedMRs =
+        new MemoryReportsWrapper(&mapped);
+    nsRefPtr<MemoryReportCallback> cb = new MemoryReportCallback();
+
+    while (NS_SUCCEEDED(e2->HasMoreElements(&more)) && more) {
+      nsCOMPtr<nsIMemoryMultiReporter> r;
+      e2->GetNext(getter_AddRefs(r));
+      r->CollectReports(cb, wrappedMRs);
+    }
+
+    // Ignore (by zeroing its amount) any reporter that is a child of another
+    // reporter.  Eg. if we have "explicit/a" and "explicit/a/b", zero the
+    // latter.  This is quadratic in the number of MAPPED reporters, but there
+    // shouldn't be many.
+    for (PRUint32 i = 0; i < mapped.Length(); i++) {
+        const nsCString &iPath = mapped[i].path;
+        for (PRUint32 j = i + 1; j < mapped.Length(); j++) {
+            const nsCString &jPath = mapped[j].path;
+            if (isParent(iPath, jPath)) {
+                mapped[j].amount = 0;
+            } else if (isParent(jPath, iPath)) {
+                mapped[i].amount = 0;
+            }
+        }
+    }
+
+    // Sum all the mapped reporters and heapUsed.
+    *aExplicit = heapUsed;
+    for (PRUint32 i = 0; i < mapped.Length(); i++) {
+        *aExplicit += mapped[i].amount;
+    }
+
+    return NS_OK;
+}
+
 NS_IMPL_ISUPPORTS1(nsMemoryReporter, nsIMemoryReporter)
 
 nsMemoryReporter::nsMemoryReporter(nsCString& process,