Bug 936236 - Send memory-pressure notification if ArrayBuffer allocation OOMs and retry allocation (r=mccr8)

dom/base/nsJSEnvironment.cpp

@@ -2971,6 +2971,16 @@ AsmJSCacheOpenEntryForWrite(JS::Handle<JSObject*> aGlobal,
                                        aHandle);
 }
 
+static void
+OnLargeAllocationFailure()
+{
+  nsCOMPtr<nsIObserverService> os =
+    mozilla::services::GetObserverService();
+  if (os) {
+    os->NotifyObservers(nullptr, "memory-pressure", MOZ_UTF16("heap-minimize"));
+  }
+}
+
 static NS_DEFINE_CID(kDOMScriptObjectFactoryCID, NS_DOM_SCRIPT_OBJECT_FACTORY_CID);
 
 void
@@ -3027,6 +3037,8 @@ nsJSContext::EnsureStatics()
   };
   JS::SetAsmJSCacheOps(sRuntime, &asmJSCacheOps);
 
+  JS::SetLargeAllocationFailureCallback(sRuntime, OnLargeAllocationFailure);
+
   // Set these global xpconnect options...
   Preferences::RegisterCallbackAndCall(ReportAllJSExceptionsPrefChangedCallback,
                                        "dom.report_all_js_exceptions");

js/src/jsapi.cpp

@@ -6297,3 +6297,9 @@ JSAutoByteString::encodeLatin1(ExclusiveContext *cx, JSString *str)
     mBytes = LossyTwoByteCharsToNewLatin1CharsZ(cx, linear->range()).c_str();
     return mBytes;
 }
+
+JS_PUBLIC_API(void)
+JS::SetLargeAllocationFailureCallback(JSRuntime *rt, JS::LargeAllocationFailureCallback lafc)
+{
+    rt->largeAllocationFailureCallback = lafc;
+}

js/src/jsapi.h

@@ -4919,6 +4919,20 @@ class MOZ_STACK_CLASS JS_PUBLIC_API(ForOfIterator) {
     bool materializeArrayIterator();
 };
 
+
+/*
+ * If a large allocation fails, the JS engine may call the large-allocation-
+ * failure callback, if set, to allow the embedding to flush caches, possibly
+ * perform shrinking GCs, etc. to make some room so that the allocation will
+ * succeed if retried.
+ */
+
+typedef void
+(* LargeAllocationFailureCallback)();
+
+extern JS_PUBLIC_API(void)
+SetLargeAllocationFailureCallback(JSRuntime *rt, LargeAllocationFailureCallback afc);
+
 } /* namespace JS */
 
 #endif /* jsapi_h */

js/src/vm/Runtime.cpp

@@ -302,10 +302,11 @@ JSRuntime::JSRuntime(JSUseHelperThreads useHelperThreads)
     useHelperThreads_(useHelperThreads),
     parallelIonCompilationEnabled_(true),
     parallelParsingEnabled_(true),
-    isWorkerRuntime_(false)
+    isWorkerRuntime_(false),
 #ifdef DEBUG
-    , enteredPolicy(nullptr)
+    enteredPolicy(nullptr),
 #endif
+    largeAllocationFailureCallback(nullptr)
 {
     liveRuntimesCount++;
 

js/src/vm/Runtime.h

@@ -1788,6 +1788,34 @@ struct JSRuntime : public JS::shadow::Runtime,
   public:
     js::AutoEnterPolicy *enteredPolicy;
 #endif
+
+    /*
+     * These variations of malloc/calloc/realloc will call the
+     * large-allocation-failure callback on OOM and retry the allocation.
+     */
+    JS::LargeAllocationFailureCallback largeAllocationFailureCallback;
+
+    static const unsigned LARGE_ALLOCATION = 25 * 1024 * 1024;
+
+    void *callocCanGC(size_t bytes) {
+        void *p = calloc_(bytes);
+        if (MOZ_LIKELY(!!p))
+            return p;
+        if (!largeAllocationFailureCallback || bytes < LARGE_ALLOCATION)
+            return nullptr;
+        largeAllocationFailureCallback();
+        return js_calloc(bytes);
+    }
+
+    void *reallocCanGC(void *p, size_t bytes) {
+        void *p2 = realloc_(p, bytes);
+        if (MOZ_LIKELY(!!p2))
+            return p2;
+        if (!largeAllocationFailureCallback || bytes < LARGE_ALLOCATION)
+            return nullptr;
+        largeAllocationFailureCallback();
+        return js_realloc(p, bytes);
+    }
 };
 
 namespace js {

js/src/vm/TypedArrayObject.cpp

@@ -241,14 +241,14 @@ AllocateArrayBufferContents(JSContext *maybecx, uint32_t nbytes, void *oldptr =
         ObjectElements *oldheader = static_cast<ObjectElements *>(oldptr);
         uint32_t oldnbytes = ArrayBufferObject::headerInitializedLength(oldheader);
 
-        void *p = maybecx ? maybecx->realloc_(oldptr, size) : js_realloc(oldptr, size);
+        void *p = maybecx ? maybecx->runtime()->reallocCanGC(oldptr, size) : js_realloc(oldptr, size);
         newheader = static_cast<ObjectElements *>(p);
 
         // if we grew the array, we need to set the new bytes to 0
         if (newheader && nbytes > oldnbytes)
             memset(reinterpret_cast<uint8_t*>(newheader->elements()) + oldnbytes, 0, nbytes - oldnbytes);
     } else {
-        void *p = maybecx ? maybecx->calloc_(size) : js_calloc(size);
+        void *p = maybecx ? maybecx->runtime()->callocCanGC(size) : js_calloc(size);
         newheader = static_cast<ObjectElements *>(p);
     }
     if (!newheader) {
@@ -257,7 +257,6 @@ AllocateArrayBufferContents(JSContext *maybecx, uint32_t nbytes, void *oldptr =
         return nullptr;
     }
 
-    // we rely on this being correct
     ArrayBufferObject::updateElementsHeader(newheader, nbytes);
 
     return newheader;