Bug 1156317 - Change the onOutOfMemory() interface is make it harder to misuse r=terrence

js/src/builtin/TestingFunctions.cpp

@@ -2110,7 +2110,7 @@ static bool
 ReportLargeAllocationFailure(JSContext* cx, unsigned argc, jsval* vp)
 {
     CallArgs args = CallArgsFromVp(argc, vp);
-    void* buf = cx->runtime()->onOutOfMemoryCanGC(NULL, JSRuntime::LARGE_ALLOCATION);
+    void* buf = cx->runtime()->onOutOfMemoryCanGC(AllocFunction::Malloc, JSRuntime::LARGE_ALLOCATION);
     js_free(buf);
     args.rval().setUndefined();
     return true;

js/src/gc/Zone.h

@@ -133,8 +133,8 @@ struct Zone : public JS::shadow::Zone,
     bool isTooMuchMalloc() const { return gcMallocBytes <= 0; }
     void onTooMuchMalloc();
 
-    void* onOutOfMemory(void* p, size_t nbytes) {
-        return runtimeFromMainThread()->onOutOfMemory(p, nbytes);
+    void* onOutOfMemory(js::AllocFunction allocFunc, size_t nbytes, void* reallocPtr = nullptr) {
+        return runtimeFromMainThread()->onOutOfMemory(allocFunc, nbytes, reallocPtr);
     }
     void reportAllocationOverflow() { js::ReportAllocationOverflow(nullptr); }
 

js/src/jsalloc.cpp

@@ -11,9 +11,9 @@
 using namespace js;
 
 void*
-TempAllocPolicy::onOutOfMemory(void* p, size_t nbytes)
+TempAllocPolicy::onOutOfMemory(AllocFunction allocFunc, size_t nbytes, void* reallocPtr)
 {
-    return static_cast<ExclusiveContext*>(cx_)->onOutOfMemory(p, nbytes);
+    return static_cast<ExclusiveContext*>(cx_)->onOutOfMemory(allocFunc, nbytes, reallocPtr);
 }
 
 void

js/src/jsalloc.h

@@ -19,6 +19,12 @@
 
 namespace js {
 
+enum class AllocFunction {
+    Malloc,
+    Calloc,
+    Realloc
+};
+
 struct ContextFriendFields;
 
 /* Policy for using system memory functions and doing no error reporting. */
@@ -51,7 +57,8 @@ class TempAllocPolicy
      * Non-inline helper to call JSRuntime::onOutOfMemory with minimal
      * code bloat.
      */
-    JS_FRIEND_API(void*) onOutOfMemory(void* p, size_t nbytes);
+    JS_FRIEND_API(void*) onOutOfMemory(AllocFunction allocFunc, size_t nbytes,
+                                       void* reallocPtr = nullptr);
 
   public:
     MOZ_IMPLICIT TempAllocPolicy(JSContext* cx) : cx_((ContextFriendFields*) cx) {} // :(
@@ -61,7 +68,7 @@ class TempAllocPolicy
     T* pod_malloc(size_t numElems) {
         T* p = js_pod_malloc<T>(numElems);
         if (MOZ_UNLIKELY(!p))
-            p = static_cast<T*>(onOutOfMemory(nullptr, numElems * sizeof(T)));
+            p = static_cast<T*>(onOutOfMemory(AllocFunction::Malloc, numElems * sizeof(T)));
         return p;
     }
 
@@ -69,7 +76,7 @@ class TempAllocPolicy
     T* pod_calloc(size_t numElems) {
         T* p = js_pod_calloc<T>(numElems);
         if (MOZ_UNLIKELY(!p))
-            p = static_cast<T*>(onOutOfMemory(reinterpret_cast<void*>(1), numElems * sizeof(T)));
+            p = static_cast<T*>(onOutOfMemory(AllocFunction::Calloc, numElems * sizeof(T)));
         return p;
     }
 
@@ -77,7 +84,7 @@ class TempAllocPolicy
     T* pod_realloc(T* prior, size_t oldSize, size_t newSize) {
         T* p2 = js_pod_realloc<T>(prior, oldSize, newSize);
         if (MOZ_UNLIKELY(!p2))
-            p2 = static_cast<T*>(onOutOfMemory(prior, newSize * sizeof(T)));
+            p2 = static_cast<T*>(onOutOfMemory(AllocFunction::Realloc, newSize * sizeof(T), prior));
         return p2;
     }
 

js/src/jscntxt.h

@@ -164,8 +164,8 @@ class ExclusiveContext : public ContextFriendFields,
         return thing->compartment() == compartment_;
     }
 
-    void* onOutOfMemory(void* p, size_t nbytes) {
-        return runtime_->onOutOfMemory(p, nbytes, maybeJSContext());
+    void* onOutOfMemory(js::AllocFunction allocFunc, size_t nbytes, void* reallocPtr = nullptr) {
+        return runtime_->onOutOfMemory(allocFunc, nbytes, reallocPtr, maybeJSContext());
     }
 
     /* Clear the pending exception (if any) due to OOM. */

js/src/vm/MallocProvider.h

@@ -68,7 +68,7 @@ struct MallocProvider
             client()->reportAllocationOverflow();
             return nullptr;
         }
-        return (T*)client()->onOutOfMemory(nullptr, numElems * sizeof(T));
+        return (T*)client()->onOutOfMemory(AllocFunction::Malloc, numElems * sizeof(T));
     }
 
     template <class T, class U>
@@ -87,7 +87,7 @@ struct MallocProvider
             client()->updateMallocCounter(bytes);
             return p;
         }
-        return (T*)client()->onOutOfMemory(nullptr, bytes);
+        return (T*)client()->onOutOfMemory(AllocFunction::Malloc, bytes);
     }
 
     template <class T>
@@ -112,7 +112,7 @@ struct MallocProvider
             client()->reportAllocationOverflow();
             return nullptr;
         }
-        return (T*)client()->onOutOfMemory(reinterpret_cast<void*>(1), numElems * sizeof(T));
+        return (T*)client()->onOutOfMemory(AllocFunction::Calloc, numElems * sizeof(T));
     }
 
     template <class T, class U>
@@ -131,7 +131,7 @@ struct MallocProvider
             client()->updateMallocCounter(bytes);
             return p;
         }
-        return (T*)client()->onOutOfMemory(reinterpret_cast<void*>(1), bytes);
+        return (T*)client()->onOutOfMemory(AllocFunction::Calloc, bytes);
     }
 
     template <class T>
@@ -155,7 +155,7 @@ struct MallocProvider
             client()->reportAllocationOverflow();
             return nullptr;
         }
-        return (T*)client()->onOutOfMemory(prior, newSize * sizeof(T));
+        return (T*)client()->onOutOfMemory(AllocFunction::Realloc, newSize * sizeof(T), prior);
     }
 
     JS_DECLARE_NEW_METHODS(new_, pod_malloc<uint8_t>, MOZ_ALWAYS_INLINE)

js/src/vm/Runtime.cpp

@@ -737,14 +737,11 @@ JSRuntime::onTooMuchMalloc()
 }
 
 JS_FRIEND_API(void*)
-JSRuntime::onOutOfMemory(void* p, size_t nbytes)
+JSRuntime::onOutOfMemory(AllocFunction allocFunc, size_t nbytes, void* reallocPtr,
+                         JSContext* maybecx)
 {
-    return onOutOfMemory(p, nbytes, nullptr);
-}
+    MOZ_ASSERT_IF(allocFunc != AllocFunction::Realloc, !reallocPtr);
 
-JS_FRIEND_API(void*)
-JSRuntime::onOutOfMemory(void* p, size_t nbytes, JSContext* cx)
-{
     if (isHeapBusy())
         return nullptr;
 
@@ -753,25 +750,34 @@ JSRuntime::onOutOfMemory(void* p, size_t nbytes, JSContext* cx)
      * all the allocations and released the empty GC chunks.
      */
     gc.onOutOfMallocMemory();
-    if (!p)
+    void* p;
+    switch (allocFunc) {
+      case AllocFunction::Malloc:
         p = js_malloc(nbytes);
-    else if (p == reinterpret_cast<void*>(1))
+        break;
+      case AllocFunction::Calloc:
         p = js_calloc(nbytes);
-    else
-        p = js_realloc(p, nbytes);
+        break;
+      case AllocFunction::Realloc:
+        p = js_realloc(reallocPtr, nbytes);
+        break;
+      default:
+        MOZ_CRASH();
+    }
     if (p)
         return p;
-    if (cx)
-        ReportOutOfMemory(cx);
+
+    if (maybecx)
+        ReportOutOfMemory(maybecx);
     return nullptr;
 }
 
 void*
-JSRuntime::onOutOfMemoryCanGC(void* p, size_t bytes)
+JSRuntime::onOutOfMemoryCanGC(AllocFunction allocFunc, size_t bytes, void* reallocPtr)
 {
     if (largeAllocationFailureCallback && bytes >= LARGE_ALLOCATION)
         largeAllocationFailureCallback(largeAllocationFailureCallbackData);
-    return onOutOfMemory(p, bytes);
+    return onOutOfMemory(allocFunc, bytes, reallocPtr);
 }
 
 bool

js/src/vm/Runtime.h

@@ -1359,18 +1359,18 @@ struct JSRuntime : public JS::shadow::Runtime,
     JS_FRIEND_API(void) onTooMuchMalloc();
 
     /*
-     * This should be called after system malloc/realloc returns nullptr to try
-     * to recove some memory or to report an error. Failures in malloc and
-     * calloc are signaled by p == null and p == reinterpret_cast<void*>(1).
-     * Other values of p mean a realloc failure.
+     * This should be called after system malloc/calloc/realloc returns nullptr
+     * to try to recove some memory or to report an error.  For realloc, the
+     * original pointer must be passed as reallocPtr.
      *
      * The function must be called outside the GC lock.
      */
-    JS_FRIEND_API(void*) onOutOfMemory(void* p, size_t nbytes);
-    JS_FRIEND_API(void*) onOutOfMemory(void* p, size_t nbytes, JSContext* cx);
+    JS_FRIEND_API(void*) onOutOfMemory(js::AllocFunction allocator, size_t nbytes,
+                                       void* reallocPtr = nullptr, JSContext* maybecx = nullptr);
 
     /*  onOutOfMemory but can call the largeAllocationFailureCallback. */
-    JS_FRIEND_API(void*) onOutOfMemoryCanGC(void* p, size_t bytes);
+    JS_FRIEND_API(void*) onOutOfMemoryCanGC(js::AllocFunction allocator, size_t nbytes,
+                                            void* reallocPtr = nullptr);
 
     void addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf, JS::RuntimeSizes* runtime);
 
@@ -1434,7 +1434,7 @@ struct JSRuntime : public JS::shadow::Runtime,
             reportAllocationOverflow();
             return nullptr;
         }
-        return (T*)onOutOfMemoryCanGC(reinterpret_cast<void*>(1), numElems * sizeof(T));
+        return (T*)onOutOfMemoryCanGC(js::AllocFunction::Calloc, numElems * sizeof(T));
     }
 
     template <typename T>
@@ -1446,7 +1446,7 @@ struct JSRuntime : public JS::shadow::Runtime,
             reportAllocationOverflow();
             return nullptr;
         }
-        return (T*)onOutOfMemoryCanGC(p, newSize * sizeof(T));
+        return (T*)onOutOfMemoryCanGC(js::AllocFunction::Realloc, newSize * sizeof(T), p);
     }
 
     /*