Bug 1203840 - Trigger dirty pages purge after CC. r=njn,r=smaug,r=mccr8

Jemalloc 4 purges dirty pages regularly during free() when the ratio of dirty
pages compared to active pages is higher than 1 << lg_dirty_mult.  We set
lg_dirty_mult in jemalloc_config to limit RSS usage, but it also has an impact
on performance.

So instead of enforcing a high ratio to force more pages being purged, we keep
jemalloc's default ratio of 8, and force a regular purge of all dirty pages,
after cycle collection.

Keeping jemalloc's default ratio avoids cycle-collection-triggered purge to
have to go through really all dirty pages when there are a lot, in which case
the normal jemalloc purge during free() will already have kicked in. It also
takes care of everything that doesn't run the cycle collector still having
a level of purge, like plugins in the plugin-container.

At the same time, since jemalloc_purge_freed_pages does nothing with jemalloc 4,
repurpose the MEMORY_FREE_PURGED_PAGES_MS telemetry probe to track the time
spent in this cycle-collector-triggered purge.

dom/workers/RuntimeService.cpp

@@ -976,7 +976,7 @@ public:
   }
 
   void
-  DispatchDeferredDeletion(bool aContinuation) override
+  DispatchDeferredDeletion(bool aContinuation, bool aPurge) override
   {
     MOZ_ASSERT(!aContinuation);
 

js/xpconnect/src/XPCJSRuntime.cpp

@@ -57,6 +57,10 @@
 #include "nsExceptionHandler.h"
 #endif
 
+#if defined(MOZ_JEMALLOC4)
+#include "mozmemory.h"
+#endif
+
 using namespace mozilla;
 using namespace xpc;
 using namespace JS;
@@ -146,26 +150,40 @@ public:
               mActive = false;
           }
       } else {
+#if defined(MOZ_JEMALLOC4)
+          if (mPurge) {
+              /* Jemalloc purges dirty pages regularly during free() when the
+               * ratio of dirty pages compared to active pages is higher than
+               * 1 << lg_dirty_mult. A high ratio can have an impact on
+               * performance, so we use the default ratio of 8, but force a
+               * regular purge of all remaining dirty pages, after cycle
+               * collection. */
+              Telemetry::AutoTimer<Telemetry::MEMORY_FREE_PURGED_PAGES_MS> timer;
+              jemalloc_free_dirty_pages();
+          }
+#endif
           mActive = false;
       }
       return NS_OK;
   }
 
-  void Dispatch(bool aContinuation = false)
+  void Dispatch(bool aContinuation = false, bool aPurge = false)
   {
       if (mContinuation) {
           mContinuation = aContinuation;
       }
+      mPurge = aPurge;
       if (!mActive && NS_SUCCEEDED(NS_DispatchToCurrentThread(this))) {
           mActive = true;
       }
   }
 
-  AsyncFreeSnowWhite() : mContinuation(false), mActive(false) {}
+  AsyncFreeSnowWhite() : mContinuation(false), mActive(false), mPurge(false) {}
 
 public:
   bool mContinuation;
   bool mActive;
+  bool mPurge;
 };
 
 namespace xpc {
@@ -630,9 +648,9 @@ XPCJSRuntime::EndCycleCollectionCallback(CycleCollectorResults& aResults)
 }
 
 void
-XPCJSRuntime::DispatchDeferredDeletion(bool aContinuation)
+XPCJSRuntime::DispatchDeferredDeletion(bool aContinuation, bool aPurge)
 {
-    mAsyncSnowWhiteFreer->Dispatch(aContinuation);
+    mAsyncSnowWhiteFreer->Dispatch(aContinuation, aPurge);
 }
 
 void

js/xpconnect/src/xpcprivate.h

@@ -554,7 +554,7 @@ public:
     void PrepareForForgetSkippable() override;
     void BeginCycleCollectionCallback() override;
     void EndCycleCollectionCallback(mozilla::CycleCollectorResults& aResults) override;
-    void DispatchDeferredDeletion(bool continuation) override;
+    void DispatchDeferredDeletion(bool aContinuation, bool aPurge = false) override;
 
     void CustomGCCallback(JSGCStatus status) override;
     void CustomOutOfMemoryCallback() override;

memory/build/jemalloc_config.cpp

@@ -22,15 +22,6 @@
 #endif
 
 /* Override some jemalloc defaults */
-#ifdef MOZ_WIDGET_GONK
-/* we tolerate around 4MiB of dirty pages on most platforms, except for B2G,
- * where our limit is 1MiB
- */
-#define MOZ_MALLOC_PLATFORM_OPTIONS ",lg_dirty_mult:8"
-#else
-#define MOZ_MALLOC_PLATFORM_OPTIONS ",lg_dirty_mult:6"
-#endif
-
 #ifdef DEBUG
 #define MOZ_MALLOC_BUILD_OPTIONS ",junk:true"
 #else
@@ -39,7 +30,7 @@
 
 #define MOZ_MALLOC_OPTIONS "narenas:1,tcache:false"
 MFBT_DATA const char* je_(malloc_conf) =
-  MOZ_MALLOC_OPTIONS MOZ_MALLOC_PLATFORM_OPTIONS MOZ_MALLOC_BUILD_OPTIONS;
+  MOZ_MALLOC_OPTIONS MOZ_MALLOC_BUILD_OPTIONS;
 
 #ifdef ANDROID
 #include <android/log.h>

toolkit/components/telemetry/Histograms.json

@@ -575,8 +575,7 @@
     "high": "1024",
     "n_buckets": 10,
     "extended_statistics_ok": true,
-    "description": "Time(ms) to purge MADV_FREE'd heap pages.",
-    "cpp_guard": "XP_MACOSX"
+    "description": "Time(ms) to purge dirty heap pages."
   },
   "LOW_MEMORY_EVENTS_VIRTUAL": {
     "alert_emails": ["memshrink-telemetry-alerts@mozilla.com"],

xpcom/base/CycleCollectedJSRuntime.h

@@ -290,7 +290,7 @@ public:
   virtual void PrepareForForgetSkippable() = 0;
   virtual void BeginCycleCollectionCallback() = 0;
   virtual void EndCycleCollectionCallback(CycleCollectorResults& aResults) = 0;
-  virtual void DispatchDeferredDeletion(bool aContinuation) = 0;
+  virtual void DispatchDeferredDeletion(bool aContinuation, bool aPurge = false) = 0;
 
   JSRuntime* Runtime() const
   {

xpcom/base/nsCycleCollector.cpp

@@ -3296,7 +3296,7 @@ nsCycleCollector::CollectWhite()
   }
   timeLog.Checkpoint("CollectWhite::Unroot");
 
-  nsCycleCollector_dispatchDeferredDeletion(false);
+  nsCycleCollector_dispatchDeferredDeletion(false, true);
   timeLog.Checkpoint("CollectWhite::dispatchDeferredDeletion");
 
   mIncrementalPhase = CleanupPhase;
@@ -4057,11 +4057,11 @@ nsCycleCollector_forgetSkippable(bool aRemoveChildlessNodes,
 }
 
 void
-nsCycleCollector_dispatchDeferredDeletion(bool aContinuation)
+nsCycleCollector_dispatchDeferredDeletion(bool aContinuation, bool aPurge)
 {
   CycleCollectedJSRuntime* rt = CycleCollectedJSRuntime::Get();
   if (rt) {
-    rt->DispatchDeferredDeletion(aContinuation);
+    rt->DispatchDeferredDeletion(aContinuation, aPurge);
   }
 }
 

xpcom/base/nsCycleCollector.h

@@ -39,7 +39,8 @@ void nsCycleCollector_prepareForGarbageCollection();
 // If an incremental cycle collection is in progress, finish it.
 void nsCycleCollector_finishAnyCurrentCollection();
 
-void nsCycleCollector_dispatchDeferredDeletion(bool aContinuation = false);
+void nsCycleCollector_dispatchDeferredDeletion(bool aContinuation = false,
+                                               bool aPurge = false);
 bool nsCycleCollector_doDeferredDeletion();
 
 already_AddRefed<nsICycleCollectorLogSink> nsCycleCollector_createLogSink();

xpcom/base/nsMemoryReporterManager.cpp

@@ -457,10 +457,12 @@ static nsresult
 ResidentDistinguishedAmountHelper(int64_t* aN, bool aDoPurge)
 {
 #ifdef HAVE_JEMALLOC_STATS
+#ifndef MOZ_JEMALLOC4
   if (aDoPurge) {
     Telemetry::AutoTimer<Telemetry::MEMORY_FREE_PURGED_PAGES_MS> timer;
     jemalloc_purge_freed_pages();
   }
+#endif
 #endif
 
   task_basic_info ti;