Bug 1207519 - Prevent HashTable shrink from ignoring allocation failures that may have been reported r=Waldo

js/public/HashTable.h

@@ -1124,11 +1124,24 @@ class HashTable : private AllocPolicy
         return keyHash & ~sCollisionBit;
     }
 
-    static Entry* createTable(AllocPolicy& alloc, uint32_t capacity)
+    enum FailureBehavior { DontReportFailure = false, ReportFailure = true };
+
+    static Entry* createTable(AllocPolicy& alloc, uint32_t capacity,
+                              FailureBehavior reportFailure = ReportFailure)
     {
         static_assert(sFreeKey == 0,
                       "newly-calloc'd tables have to be considered empty");
-        return alloc.template pod_calloc<Entry>(capacity);
+        if (reportFailure)
+            return alloc.template pod_calloc<Entry>(capacity);
+
+        return alloc.template maybe_pod_calloc<Entry>(capacity);
+    }
+
+    static Entry* maybeCreateTable(AllocPolicy& alloc, uint32_t capacity)
+    {
+        static_assert(sFreeKey == 0,
+                      "newly-calloc'd tables have to be considered empty");
+        return alloc.template maybe_pod_calloc<Entry>(capacity);
     }
 
     static void destroyTable(AllocPolicy& alloc, Entry* oldTable, uint32_t capacity)
@@ -1367,7 +1380,7 @@ class HashTable : private AllocPolicy
 
     enum RebuildStatus { NotOverloaded, Rehashed, RehashFailed };
 
-    RebuildStatus changeTableSize(int deltaLog2)
+    RebuildStatus changeTableSize(int deltaLog2, FailureBehavior reportFailure = ReportFailure)
     {
         // Look, but don't touch, until we succeed in getting new entry store.
         Entry* oldTable = table;
@@ -1375,11 +1388,12 @@ class HashTable : private AllocPolicy
         uint32_t newLog2 = sHashBits - hashShift + deltaLog2;
         uint32_t newCapacity = JS_BIT(newLog2);
         if (MOZ_UNLIKELY(newCapacity > sMaxCapacity)) {
-            this->reportAllocOverflow();
+            if (reportFailure)
+                this->reportAllocOverflow();
             return RehashFailed;
         }
 
-        Entry* newTable = createTable(*this, newCapacity);
+        Entry* newTable = createTable(*this, newCapacity, reportFailure);
         if (!newTable)
             return RehashFailed;
 
@@ -1405,14 +1419,19 @@ class HashTable : private AllocPolicy
         return Rehashed;
     }
 
-    RebuildStatus checkOverloaded()
+    bool shouldCompressTable()
+    {
+        // Compress if a quarter or more of all entries are removed.
+        return removedCount >= (capacity() >> 2);
+    }
+
+    RebuildStatus checkOverloaded(FailureBehavior reportFailure = ReportFailure)
     {
         if (!overloaded())
             return NotOverloaded;
 
-        // Compress if a quarter or more of all entries are removed.
         int deltaLog2;
-        if (removedCount >= (capacity() >> 2)) {
+        if (shouldCompressTable()) {
             METER(stats.compresses++);
             deltaLog2 = 0;
         } else {
@@ -1420,14 +1439,14 @@ class HashTable : private AllocPolicy
             deltaLog2 = 1;
         }
 
-        return changeTableSize(deltaLog2);
+        return changeTableSize(deltaLog2, reportFailure);
     }
 
     // Infallibly rehash the table if we are overloaded with removals.
     void checkOverRemoved()
     {
         if (overloaded()) {
-            if (checkOverloaded() == RehashFailed)
+            if (checkOverloaded(DontReportFailure) == RehashFailed)
                 rehashTableInPlace();
         }
     }
@@ -1454,7 +1473,7 @@ class HashTable : private AllocPolicy
     {
         if (underloaded()) {
             METER(stats.shrinks++);
-            (void) changeTableSize(-1);
+            (void) changeTableSize(-1, DontReportFailure);
         }
     }
 
@@ -1470,9 +1489,8 @@ class HashTable : private AllocPolicy
             resizeLog2--;
         }
 
-        if (resizeLog2 != 0) {
+        if (resizeLog2 != 0)
-            changeTableSize(resizeLog2);
+            (void) changeTableSize(resizeLog2, DontReportFailure);
-        }
     }
 
     // This is identical to changeTableSize(currentSize), but without requiring

js/src/ds/LifoAlloc.h

@@ -531,7 +531,7 @@ class LifoAllocPolicy
       : alloc_(alloc)
     {}
     template <typename T>
-    T* pod_malloc(size_t numElems) {
+    T* maybe_pod_malloc(size_t numElems) {
         if (MOZ_UNLIKELY(numElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value))
             return nullptr;
         size_t bytes = numElems * sizeof(T);
@@ -539,7 +539,7 @@ class LifoAllocPolicy
         return static_cast<T*>(p);
     }
     template <typename T>
-    T* pod_calloc(size_t numElems) {
+    T* maybe_pod_calloc(size_t numElems) {
         T* p = pod_malloc<T>(numElems);
         if (fb == Fallible && !p)
             return nullptr;
@@ -547,7 +547,7 @@ class LifoAllocPolicy
         return p;
     }
     template <typename T>
-    T* pod_realloc(T* p, size_t oldSize, size_t newSize) {
+    T* maybe_pod_realloc(T* p, size_t oldSize, size_t newSize) {
         T* n = pod_malloc<T>(newSize);
         if (fb == Fallible && !n)
             return nullptr;
@@ -555,6 +555,18 @@ class LifoAllocPolicy
         memcpy(n, p, Min(oldSize * sizeof(T), newSize * sizeof(T)));
         return n;
     }
+    template <typename T>
+    T* pod_malloc(size_t numElems) {
+        return maybe_pod_malloc<T>(numElems);
+    }
+    template <typename T>
+    T* pod_calloc(size_t numElems) {
+        return maybe_pod_calloc<T>(numElems);
+    }
+    template <typename T>
+    T* pod_realloc(T* p, size_t oldSize, size_t newSize) {
+        return maybe_pod_realloc<T>(p, oldSize, newSize);
+    }
     void free_(void* p) {
     }
     void reportAllocOverflow() const {

js/src/jit/JitAllocPolicy.h

@@ -78,20 +78,20 @@ class JitAllocPolicy
       : alloc_(alloc)
     {}
     template <typename T>
-    T* pod_malloc(size_t numElems) {
+    T* maybe_pod_malloc(size_t numElems) {
         if (MOZ_UNLIKELY(numElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value))
             return nullptr;
         return static_cast<T*>(alloc_.allocate(numElems * sizeof(T)));
     }
     template <typename T>
-    T* pod_calloc(size_t numElems) {
+    T* maybe_pod_calloc(size_t numElems) {
         T* p = pod_malloc<T>(numElems);
         if (MOZ_LIKELY(p))
             memset(p, 0, numElems * sizeof(T));
         return p;
     }
     template <typename T>
-    T* pod_realloc(T* p, size_t oldSize, size_t newSize) {
+    T* maybe_pod_realloc(T* p, size_t oldSize, size_t newSize) {
         T* n = pod_malloc<T>(newSize);
         if (MOZ_UNLIKELY(!n))
             return n;
@@ -99,6 +99,18 @@ class JitAllocPolicy
         memcpy(n, p, Min(oldSize * sizeof(T), newSize * sizeof(T)));
         return n;
     }
+    template <typename T>
+    T* pod_malloc(size_t numElems) {
+        return maybe_pod_malloc<T>(numElems);
+    }
+    template <typename T>
+    T* pod_calloc(size_t numElems) {
+        return maybe_pod_calloc<T>(numElems);
+    }
+    template <typename T>
+    T* pod_realloc(T* ptr, size_t oldSize, size_t newSize) {
+        return maybe_pod_realloc<T>(ptr, oldSize, newSize);
+    }
     void free_(void* p) {
     }
     void reportAllocOverflow() const {
@@ -114,11 +126,15 @@ class OldJitAllocPolicy
     OldJitAllocPolicy()
     {}
     template <typename T>
-    T* pod_malloc(size_t numElems) {
+    T* maybe_pod_malloc(size_t numElems) {
         if (MOZ_UNLIKELY(numElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value))
             return nullptr;
         return static_cast<T*>(GetJitContext()->temp->allocate(numElems * sizeof(T)));
     }
+    template <typename T>
+    T* pod_malloc(size_t numElems) {
+        return maybe_pod_malloc<T>(numElems);
+    }
     void free_(void* p) {
     }
     void reportAllocOverflow() const {

js/src/jsalloc.h

@@ -31,11 +31,16 @@ struct ContextFriendFields;
 class SystemAllocPolicy
 {
   public:
-    template <typename T> T* pod_malloc(size_t numElems) { return js_pod_malloc<T>(numElems); }
+    template <typename T> T* maybe_pod_malloc(size_t numElems) { return js_pod_malloc<T>(numElems); }
-    template <typename T> T* pod_calloc(size_t numElems) { return js_pod_calloc<T>(numElems); }
+    template <typename T> T* maybe_pod_calloc(size_t numElems) { return js_pod_calloc<T>(numElems); }
-    template <typename T> T* pod_realloc(T* p, size_t oldSize, size_t newSize) {
+    template <typename T> T* maybe_pod_realloc(T* p, size_t oldSize, size_t newSize) {
         return js_pod_realloc<T>(p, oldSize, newSize);
     }
+    template <typename T> T* pod_malloc(size_t numElems) { return maybe_pod_malloc<T>(numElems); }
+    template <typename T> T* pod_calloc(size_t numElems) { return maybe_pod_calloc<T>(numElems); }
+    template <typename T> T* pod_realloc(T* p, size_t oldSize, size_t newSize) {
+        return maybe_pod_realloc<T>(p, oldSize, newSize);
+    }
     void free_(void* p) { js_free(p); }
     void reportAllocOverflow() const {}
     bool checkSimulatedOOM() const {
@@ -70,9 +75,24 @@ class TempAllocPolicy
     MOZ_IMPLICIT TempAllocPolicy(JSContext* cx) : cx_((ContextFriendFields*) cx) {} // :(
     MOZ_IMPLICIT TempAllocPolicy(ContextFriendFields* cx) : cx_(cx) {}
 
+    template <typename T>
+    T* maybe_pod_malloc(size_t numElems) {
+        return js_pod_malloc<T>(numElems);
+    }
+
+    template <typename T>
+    T* maybe_pod_calloc(size_t numElems) {
+        return js_pod_calloc<T>(numElems);
+    }
+
+    template <typename T>
+    T* maybe_pod_realloc(T* prior, size_t oldSize, size_t newSize) {
+        return js_pod_realloc<T>(prior, oldSize, newSize);
+    }
+
     template <typename T>
     T* pod_malloc(size_t numElems) {
-        T* p = js_pod_malloc<T>(numElems);
+        T* p = maybe_pod_malloc<T>(numElems);
         if (MOZ_UNLIKELY(!p))
             p = static_cast<T*>(onOutOfMemory(AllocFunction::Malloc, numElems * sizeof(T)));
         return p;
@@ -80,7 +100,7 @@ class TempAllocPolicy
 
     template <typename T>
     T* pod_calloc(size_t numElems) {
-        T* p = js_pod_calloc<T>(numElems);
+        T* p = maybe_pod_calloc<T>(numElems);
         if (MOZ_UNLIKELY(!p))
             p = static_cast<T*>(onOutOfMemory(AllocFunction::Calloc, numElems * sizeof(T)));
         return p;
@@ -88,7 +108,7 @@ class TempAllocPolicy
 
     template <typename T>
     T* pod_realloc(T* prior, size_t oldSize, size_t newSize) {
-        T* p2 = js_pod_realloc<T>(prior, oldSize, newSize);
+        T* p2 = maybe_pod_realloc<T>(prior, oldSize, newSize);
         if (MOZ_UNLIKELY(!p2))
             p2 = static_cast<T*>(onOutOfMemory(AllocFunction::Realloc, newSize * sizeof(T), prior));
         return p2;

js/src/vm/MallocProvider.h

@@ -23,8 +23,6 @@
  *       - TempAllocPolicy: Adds automatic error reporting to the provided
  *         Context when allocations fail.
  *
- *       - ContextAllocPolicy: forwards to the JSContext MallocProvider.
- *
  *       - RuntimeAllocPolicy: forwards to the JSRuntime MallocProvider.
  *
  *   - MallocProvider. A mixin base class that handles automatically updating
@@ -52,6 +50,36 @@ namespace js {
 template<class Client>
 struct MallocProvider
 {
+    template <class T>
+    T* maybe_pod_malloc(size_t numElems) {
+        size_t bytes = numElems * sizeof(T);
+        T* p = js_pod_malloc<T>(numElems);
+        if (MOZ_LIKELY(p))
+            client()->updateMallocCounter(bytes);
+        return p;
+    }
+
+    template <class T>
+    T* maybe_pod_calloc(size_t numElems) {
+        size_t bytes = numElems * sizeof(T);
+        T* p = js_pod_calloc<T>(numElems);
+        if (MOZ_LIKELY(p))
+            client()->updateMallocCounter(bytes);
+        return p;
+    }
+
+    template <class T>
+    T* maybe_pod_realloc(T* prior, size_t oldSize, size_t newSize) {
+        T* p = js_pod_realloc(prior, oldSize, newSize);
+        if (MOZ_LIKELY(p)) {
+            // For compatibility we do not account for realloc that decreases
+            // previously allocated memory.
+            if (newSize > oldSize)
+                client()->updateMallocCounter((newSize - oldSize) * sizeof(T));
+        }
+        return p;
+    }
+
     template <class T>
     T* pod_malloc() {
         return pod_malloc<T>(1);
@@ -59,16 +87,14 @@ struct MallocProvider
 
     template <class T>
     T* pod_malloc(size_t numElems) {
-        size_t bytes = numElems * sizeof(T);
+        T* p = maybe_pod_malloc<T>(numElems);
-        T* p = js_pod_malloc<T>(numElems);
+        if (MOZ_LIKELY(p))
-        if (MOZ_LIKELY(p)) {
-            client()->updateMallocCounter(bytes);
             return p;
-        }
         if (numElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value) {
             client()->reportAllocationOverflow();
             return nullptr;
         }
+        size_t bytes = numElems * sizeof(T);
         p = (T*)client()->onOutOfMemory(AllocFunction::Malloc, bytes);
         if (p)
             client()->updateMallocCounter(bytes);
@@ -110,16 +136,14 @@ struct MallocProvider
 
     template <class T>
     T* pod_calloc(size_t numElems) {
-        size_t bytes = numElems * sizeof(T);
+        T* p = maybe_pod_calloc<T>(numElems);
-        T* p = js_pod_calloc<T>(numElems);
+        if (MOZ_LIKELY(p))
-        if (MOZ_LIKELY(p)) {
-            client()->updateMallocCounter(bytes);
             return p;
-        }
         if (numElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value) {
             client()->reportAllocationOverflow();
             return nullptr;
         }
+        size_t bytes = numElems * sizeof(T);
         p = (T*)client()->onOutOfMemory(AllocFunction::Calloc, bytes);
         if (p)
             client()->updateMallocCounter(bytes);
@@ -157,14 +181,9 @@ struct MallocProvider
 
     template <class T>
     T* pod_realloc(T* prior, size_t oldSize, size_t newSize) {
-        T* p = js_pod_realloc(prior, oldSize, newSize);
+        T* p = maybe_pod_realloc(prior, oldSize, newSize);
-        if (MOZ_LIKELY(p)) {
+        if (MOZ_LIKELY(p))
-            // For compatibility we do not account for realloc that decreases
-            // previously allocated memory.
-            if (newSize > oldSize)
-                client()->updateMallocCounter((newSize - oldSize) * sizeof(T));
             return p;
-        }
         if (newSize & mozilla::tl::MulOverflowMask<sizeof(T)>::value) {
             client()->reportAllocationOverflow();
             return nullptr;

js/src/vm/Runtime.h

@@ -2070,6 +2070,21 @@ class RuntimeAllocPolicy
   public:
     MOZ_IMPLICIT RuntimeAllocPolicy(JSRuntime* rt) : runtime(rt) {}
 
+    template <typename T>
+    T* maybe_pod_malloc(size_t numElems) {
+        return runtime->maybe_pod_malloc<T>(numElems);
+    }
+
+    template <typename T>
+    T* maybe_pod_calloc(size_t numElems) {
+        return runtime->maybe_pod_calloc<T>(numElems);
+    }
+
+    template <typename T>
+    T* maybe_pod_realloc(T* p, size_t oldSize, size_t newSize) {
+        return runtime->maybe_pod_realloc<T>(p, oldSize, newSize);
+    }
+
     template <typename T>
     T* pod_malloc(size_t numElems) {
         return runtime->pod_malloc<T>(numElems);

layout/style/nsNthIndexCache.h

@@ -63,10 +63,15 @@ private:
     void *malloc_(size_t bytes) { return ::malloc(bytes); }
 
     template <typename T>
-    T *pod_calloc(size_t numElems) {
+    T *maybe_pod_calloc(size_t numElems) {
       return static_cast<T *>(::calloc(numElems, sizeof(T)));
     }
 
+    template <typename T>
+    T *pod_calloc(size_t numElems) {
+      return maybe_pod_calloc<T>(numElems);
+    }
+
     void *realloc_(void *p, size_t bytes) { return ::realloc(p, bytes); }
     void free_(void *p) { ::free(p); }
     void reportAllocOverflow() const {}

memory/replace/dmd/DMD.cpp

@@ -134,6 +134,28 @@ class InfallibleAllocPolicy
   static void ExitOnFailure(const void* aP);
 
 public:
+  template <typename T>
+  static T* maybe_pod_malloc(size_t aNumElems)
+  {
+    if (aNumElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value)
+      return nullptr;
+    return (T*)gMallocTable->malloc(aNumElems * sizeof(T));
+  }
+
+  template <typename T>
+  static T* maybe_pod_calloc(size_t aNumElems)
+  {
+    return (T*)gMallocTable->calloc(aNumElems, sizeof(T));
+  }
+
+  template <typename T>
+  static T* maybe_pod_realloc(T* aPtr, size_t aOldSize, size_t aNewSize)
+  {
+    if (aNewSize & mozilla::tl::MulOverflowMask<sizeof(T)>::value)
+      return nullptr;
+    return (T*)gMallocTable->realloc(aPtr, aNewSize * sizeof(T));
+  }
+
   static void* malloc_(size_t aSize)
   {
     void* p = gMallocTable->malloc(aSize);
@@ -144,11 +166,9 @@ public:
   template <typename T>
   static T* pod_malloc(size_t aNumElems)
   {
-    if (aNumElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value)
+    T* p = maybe_pod_malloc<T>(aNumElems);
-      return nullptr;
-    void* p = gMallocTable->malloc(aNumElems * sizeof(T));
     ExitOnFailure(p);
-    return (T*)p;
+    return p;
   }
 
   static void* calloc_(size_t aSize)
@@ -161,9 +181,9 @@ public:
   template <typename T>
   static T* pod_calloc(size_t aNumElems)
   {
-    void* p = gMallocTable->calloc(aNumElems, sizeof(T));
+    T* p = maybe_pod_calloc<T>(aNumElems);
     ExitOnFailure(p);
-    return (T*)p;
+    return p;
   }
 
   // This realloc_ is the one we use for direct reallocs within DMD.
@@ -178,9 +198,9 @@ public:
   template <typename T>
   static T* pod_realloc(T* aPtr, size_t aOldSize, size_t aNewSize)
   {
-    if (aNewSize & mozilla::tl::MulOverflowMask<sizeof(T)>::value)
+    T* p = maybe_pod_realloc(aPtr, aOldSize, aNewSize);
-      return nullptr;
+    ExitOnFailure(p);
-    return (T*)InfallibleAllocPolicy::realloc_((void *)aPtr, aNewSize * sizeof(T));
+    return p;
   }
 
   static void* memalign_(size_t aAlignment, size_t aSize)

mfbt/AllocPolicy.h

@@ -26,6 +26,13 @@ namespace mozilla {
  * mechanism when OOM occurs.  The concept modeled here is as follows:
  *
  *  - public copy constructor, assignment, destructor
+ *  - template <typename T> T* maybe_pod_malloc(size_t)
+ *      Fallible, but doesn't report an error on OOM.
+ *  - template <typename T> T* maybe_pod_calloc(size_t)
+ *      Fallible, but doesn't report an error on OOM.
+ *  - template <typename T> T* maybe_pod_realloc(T*, size_t, size_t)
+ *      Fallible, but doesn't report an error on OOM.  The old allocation
+ *      size is passed in, in addition to the new allocation size requested.
  *  - template <typename T> T* pod_malloc(size_t)
  *      Responsible for OOM reporting when null is returned.
  *  - template <typename T> T* pod_calloc(size_t)
@@ -64,7 +71,7 @@ class MallocAllocPolicy
 {
 public:
   template <typename T>
-  T* pod_malloc(size_t aNumElems)
+  T* maybe_pod_malloc(size_t aNumElems)
   {
     if (aNumElems & mozilla::tl::MulOverflowMask<sizeof(T)>::value) {
       return nullptr;
@@ -73,13 +80,13 @@ public:
   }
 
   template <typename T>
-  T* pod_calloc(size_t aNumElems)
+  T* maybe_pod_calloc(size_t aNumElems)
   {
     return static_cast<T*>(calloc(aNumElems, sizeof(T)));
   }
 
   template <typename T>
-  T* pod_realloc(T* aPtr, size_t aOldSize, size_t aNewSize)
+  T* maybe_pod_realloc(T* aPtr, size_t aOldSize, size_t aNewSize)
   {
     if (aNewSize & mozilla::tl::MulOverflowMask<sizeof(T)>::value) {
       return nullptr;
@@ -87,6 +94,24 @@ public:
     return static_cast<T*>(realloc(aPtr, aNewSize * sizeof(T)));
   }
 
+  template <typename T>
+  T* pod_malloc(size_t aNumElems)
+  {
+    return maybe_pod_malloc<T>(aNumElems);
+  }
+
+  template <typename T>
+  T* pod_calloc(size_t aNumElems)
+  {
+    return maybe_pod_calloc<T>(aNumElems);
+  }
+
+  template <typename T>
+  T* pod_realloc(T* aPtr, size_t aOldSize, size_t aNewSize)
+  {
+    return maybe_pod_realloc<T>(aPtr, aOldSize, aNewSize);
+  }
+
   void free_(void* aPtr)
   {
     free(aPtr);