Bug 1252375 - Fix up jemalloc stats reporting. r=erahm.

We have some oddities in our jemalloc stats reporting.

- "heap-overhead-ratio" is a strange measurement: overhead / non-overhead,
  expressed as a percentage. And it omits "bin_unused", which appears to be an
  oversight.

- "heap-committed" also omits "bin_unused".

- There are some minor errors in memory report descriptions.

This patch fixes these and improves the heap reporting. It makes the following
reporting changes:

- "heap-allocated": Duplicated as "heap-committed/allocated". (We keep
  "heap-allocated" because that's a special value used in the computation of
  "heap-unclassified".)

- "heap-committed/overhead": Added; it's the same as the sum of the
  "explicit/heap-overhead/*" values. Together with "heap-committed/allocated"
  it shows clearly what fraction of the heap is overhead and what fraction is
  useful.

- "heap-committed": Removed; now implicit as the "heap-committed/" node.

- "heap-overhead-ratio":
  - Removed from memory reports; now shown as the percentage of the new
    "heap-committed/overhead" node.
  - Still available as a distinguished amount (because it's useful in
    isolation) but renamed to heapOverheadFraction, and the telemetry ID is
    renamed as MEMORY_HEAP_OVERHEAD_FRACTION.

- "heap-chunks": Removed; it's not that interesting, and can be manually
  computed as "heap-mapped" / "heap-chunksize" if necessary.

toolkit/components/aboutmemory/tests/test_memoryReporters.xul

@@ -151,7 +151,7 @@
     "residentPeak",
     "residentUnique",
     "heapAllocated",
-    "heapOverheadRatio",
+    "heapOverheadFraction",
     "JSMainRuntimeGCHeap",
     "JSMainRuntimeTemporaryPeak",
     "JSMainRuntimeCompartmentsSystem",

toolkit/components/telemetry/Histograms.json

@@ -663,13 +663,14 @@
     "n_buckets": 50,
     "description": "Committed, unused heap memory (KB)"
   },
-  "MEMORY_HEAP_COMMITTED_UNUSED_RATIO": {
+  "MEMORY_HEAP_OVERHEAD_FRACTION": {
     "alert_emails": ["memshrink-telemetry-alerts@mozilla.com"],
+    "bug_numbers": [1252375],
     "expires_in_version": "never",
     "kind": "linear",
     "high": 100,
     "n_buckets": 25,
-    "description": "Ratio of committed, unused memory to allocated memory in the heap (percentage)."
+    "description": "Fraction of committed heap memory that is overhead (percentage)."
   },
   "GHOST_WINDOWS": {
     "alert_emails": ["memshrink-telemetry-alerts@mozilla.com"],

toolkit/components/telemetry/TelemetrySession.jsm

@@ -1077,7 +1077,7 @@ var Impl = {
     b("MEMORY_RESIDENT_FAST", "residentFast");
     b("MEMORY_UNIQUE", "residentUnique");
     b("MEMORY_HEAP_ALLOCATED", "heapAllocated");
-    p("MEMORY_HEAP_COMMITTED_UNUSED_RATIO", "heapOverheadRatio");
+    p("MEMORY_HEAP_OVERHEAD_FRACTION", "heapOverheadFraction");
     b("MEMORY_JS_GC_HEAP", "JSMainRuntimeGCHeap");
     b("MEMORY_JS_MAIN_RUNTIME_TEMPORARY_PEAK", "JSMainRuntimeTemporaryPeak");
     c("MEMORY_JS_COMPARTMENTS_SYSTEM", "JSMainRuntimeCompartmentsSystem");

xpcom/base/nsIMemoryReporter.idl

@@ -345,8 +345,8 @@ interface nsIMemoryReporterManager : nsISupports
    *
    * |heapAllocated| (UNITS_BYTES)  Memory mapped by the heap allocator.
    *
-   * |heapOverheadRatio| (UNITS_PERCENTAGE)  In the heap allocator, this is the
-   * ratio of committed, unused bytes to allocated bytes.  Like all
+   * |heapOverheadFraction| (UNITS_PERCENTAGE)  In the heap allocator, this is
+   * the fraction of committed heap bytes that are overhead. Like all
    * UNITS_PERCENTAGE measurements, its amount is multiplied by 100x so it can
    * be represented by an int64_t.
    *
@@ -381,7 +381,7 @@ interface nsIMemoryReporterManager : nsISupports
   readonly attribute int64_t residentUnique;
 
   readonly attribute int64_t heapAllocated;
-  readonly attribute int64_t heapOverheadRatio;
+  readonly attribute int64_t heapOverheadFraction;
 
   readonly attribute int64_t JSMainRuntimeGCHeap;
   readonly attribute int64_t JSMainRuntimeTemporaryPeak;

xpcom/base/nsMemoryReporterManager.cpp

@@ -1275,13 +1275,21 @@ NS_IMPL_ISUPPORTS(PageFaultsHardReporter, nsIMemoryReporter)
 
 #ifdef HAVE_JEMALLOC_STATS
 
-// This has UNITS_PERCENTAGE, so it is multiplied by 100.
-static int64_t
-HeapOverheadRatio(jemalloc_stats_t* aStats)
+static size_t
+HeapOverhead(jemalloc_stats_t* aStats)
 {
-  return (int64_t)10000 *
-    (aStats->waste + aStats->bookkeeping + aStats->page_cache) /
-    ((double)aStats->allocated);
+  return aStats->waste + aStats->bookkeeping +
+         aStats->page_cache + aStats->bin_unused;
+}
+
+// This has UNITS_PERCENTAGE, so it is multiplied by 100x *again* on top of the
+// 100x for the percentage.
+static int64_t
+HeapOverheadFraction(jemalloc_stats_t* aStats)
+{
+  size_t heapOverhead = HeapOverhead(aStats);
+  size_t heapCommitted = aStats->allocated + heapOverhead;
+  return int64_t(10000 * (heapOverhead / (double)heapCommitted));
 }
 
 class JemallocHeapReporter final : public nsIMemoryReporter
@@ -1300,33 +1308,37 @@ public:
     nsresult rv;
 
     rv = MOZ_COLLECT_REPORT(
-      "heap-allocated", KIND_OTHER, UNITS_BYTES, stats.allocated,
+      "heap-committed/allocated", KIND_OTHER, UNITS_BYTES, stats.allocated,
 "Memory mapped by the heap allocator that is currently allocated to the "
 "application.  This may exceed the amount of memory requested by the "
 "application because the allocator regularly rounds up request sizes. (The "
 "exact amount requested is not recorded.)");
     NS_ENSURE_SUCCESS(rv, rv);
 
+    rv = MOZ_COLLECT_REPORT(
+      "heap-allocated", KIND_OTHER, UNITS_BYTES, stats.allocated,
+"The same as 'heap-committed/allocated'.");
+    NS_ENSURE_SUCCESS(rv, rv);
+
     // We mark this and the other heap-overhead reporters as KIND_NONHEAP
     // because KIND_HEAP memory means "counted in heap-allocated", which
     // this is not.
     rv = MOZ_COLLECT_REPORT(
       "explicit/heap-overhead/bin-unused", KIND_NONHEAP, UNITS_BYTES,
       stats.bin_unused,
-"Bytes reserved for bins of fixed-size allocations which do not correspond to "
-"an active allocation.");
+"Unused bytes due to fragmentation in the bins used for 'small' (<= 2 KiB) "
+"allocations. These bytes will be used if additional allocations occur.");
     NS_ENSURE_SUCCESS(rv, rv);
 
-    rv = MOZ_COLLECT_REPORT(
-      "explicit/heap-overhead/waste", KIND_NONHEAP, UNITS_BYTES,
-      stats.waste,
+    if (stats.waste > 0) {
+      rv = MOZ_COLLECT_REPORT(
+        "explicit/heap-overhead/waste", KIND_NONHEAP, UNITS_BYTES,
+        stats.waste,
 "Committed bytes which do not correspond to an active allocation and which the "
-"allocator is not intentionally keeping alive (i.e., not 'heap-bookkeeping' or "
-"'heap-page-cache' or 'heap-bin-unused').  Although the allocator will waste "
-"some space under any circumstances, a large value here may indicate that the "
-"heap is highly fragmented, or that allocator is performing poorly for some "
-"other reason.");
-    NS_ENSURE_SUCCESS(rv, rv);
+"allocator is not intentionally keeping alive (i.e., not "
+"'explicit/heap-overhead/{bookkeeping,page-cache,bin-unused}').");
+      NS_ENSURE_SUCCESS(rv, rv);
+    }
 
     rv = MOZ_COLLECT_REPORT(
       "explicit/heap-overhead/bookkeeping", KIND_NONHEAP, UNITS_BYTES,
@@ -1344,26 +1356,15 @@ public:
     NS_ENSURE_SUCCESS(rv, rv);
 
     rv = MOZ_COLLECT_REPORT(
-      "heap-committed", KIND_OTHER, UNITS_BYTES,
-      stats.allocated + stats.waste + stats.bookkeeping + stats.page_cache,
-"Memory mapped by the heap allocator that is committed, i.e. in physical "
-"memory or paged to disk.  This value corresponds to 'heap-allocated' + "
-"'heap-waste' + 'heap-bookkeeping' + 'heap-page-cache', but because "
-"these values are read at different times, the result probably won't match "
-"exactly.");
-    NS_ENSURE_SUCCESS(rv, rv);
-
-    rv = MOZ_COLLECT_REPORT(
-      "heap-overhead-ratio", KIND_OTHER, UNITS_PERCENTAGE,
-      HeapOverheadRatio(&stats),
-"Ratio of committed, unused bytes to allocated bytes; i.e., "
-"'heap-overhead' / 'heap-allocated'.  This measures the overhead of "
-"the heap allocator relative to amount of memory allocated.");
+      "heap-committed/overhead", KIND_OTHER, UNITS_BYTES,
+      HeapOverhead(&stats),
+"The sum of 'explicit/heap-overhead/*'.");
     NS_ENSURE_SUCCESS(rv, rv);
 
     rv = MOZ_COLLECT_REPORT(
       "heap-mapped", KIND_OTHER, UNITS_BYTES, stats.mapped,
-      "Amount of memory currently mapped.");
+"Amount of memory currently mapped. Includes memory that is uncommitted, i.e. "
+"neither in physical memory nor paged to disk.");
     NS_ENSURE_SUCCESS(rv, rv);
 
     rv = MOZ_COLLECT_REPORT(
@@ -1371,11 +1372,6 @@ public:
       "Size of chunks.");
     NS_ENSURE_SUCCESS(rv, rv);
 
-    rv = MOZ_COLLECT_REPORT(
-      "heap-chunks", KIND_OTHER, UNITS_COUNT, (stats.mapped / stats.chunksize),
-      "Number of chunks currently mapped.");
-    NS_ENSURE_SUCCESS(rv, rv);
-
     return NS_OK;
   }
 };
@@ -2388,12 +2384,12 @@ nsMemoryReporterManager::GetHeapAllocated(int64_t* aAmount)
 
 // This has UNITS_PERCENTAGE, so it is multiplied by 100x.
 NS_IMETHODIMP
-nsMemoryReporterManager::GetHeapOverheadRatio(int64_t* aAmount)
+nsMemoryReporterManager::GetHeapOverheadFraction(int64_t* aAmount)
 {
 #ifdef HAVE_JEMALLOC_STATS
   jemalloc_stats_t stats;
   jemalloc_stats(&stats);
-  *aAmount = HeapOverheadRatio(&stats);
+  *aAmount = HeapOverheadFraction(&stats);
   return NS_OK;
 #else
   *aAmount = 0;