Bug 937132 - SpiderMonkey: Check for overflows in LifoAlloc. r=luke

js/src/ds/LifoAlloc.h

@@ -305,17 +305,16 @@ class LifoAlloc
 
     template <typename T>
     T *newArray(size_t count) {
-        void *mem = alloc(sizeof(T) * count);
-        if (!mem)
-            return nullptr;
         JS_STATIC_ASSERT(mozilla::IsPod<T>::value);
-        return (T *) mem;
+        return newArrayUninitialized<T>(count);
     }
 
     // Create an array with uninitialized elements of type |T|.
     // The caller is responsible for initialization.
     template <typename T>
     T *newArrayUninitialized(size_t count) {
+        if (count & mozilla::tl::MulOverflowMask<sizeof(T)>::value)
+            return nullptr;
         return static_cast<T *>(alloc(sizeof(T) * count));
     }
 

js/src/jit/FixedList.h

@@ -28,7 +28,7 @@ class FixedList
 
   public:
     FixedList()
-      : length_(0)
+      : length_(0), list_(nullptr)
     { }
 
     // Dynamic memory allocation requires the ability to report failure.
@@ -37,6 +37,8 @@ class FixedList
         if (length == 0)
             return true;
 
+        if (length & mozilla::tl::MulOverflowMask<sizeof(T)>::value)
+            return false;
         list_ = (T *)alloc.allocate(length * sizeof(T));
         return list_ != nullptr;
     }
@@ -51,6 +53,11 @@ class FixedList
     }
 
     bool growBy(TempAllocator &alloc, size_t num) {
+        size_t newlength = length_ + num;
+        if (newlength < length_)
+            return false;
+        if (newlength & mozilla::tl::MulOverflowMask<sizeof(T)>::value)
+            return false;
         T *list = (T *)alloc.allocate((length_ + num) * sizeof(T));
         if (!list)
             return false;

js/src/jit/MIRGenerator.h

@@ -58,6 +58,8 @@ class MIRGenerator
 
     template <typename T>
     T * allocate(size_t count = 1) {
+        if (count & mozilla::tl::MulOverflowMask<sizeof(T)>::value)
+            return nullptr;
         return reinterpret_cast<T *>(alloc().allocate(sizeof(T) * count));
     }
 

js/src/jit/MIRGraph.h

@@ -561,11 +561,6 @@ class MIRGraph
         return *alloc_;
     }
 
-    template <typename T>
-    T * allocate(size_t count = 1) {
-        return reinterpret_cast<T *>(alloc_->allocate(sizeof(T) * count));
-    }
-
     void addBlock(MBasicBlock *block);
     void insertBlockAfter(MBasicBlock *at, MBasicBlock *block);
 

js/src/jsinferinlines.h

@@ -821,17 +821,19 @@ TypeCompartment::compartment()
  * probing.  TODO: replace these with jshashtables.
  */
 const unsigned SET_ARRAY_SIZE = 8;
+const unsigned SET_CAPACITY_OVERFLOW = 1u << 30;
 
 /* Get the capacity of a set with the given element count. */
 static inline unsigned
 HashSetCapacity(unsigned count)
 {
     JS_ASSERT(count >= 2);
+    JS_ASSERT(count < SET_CAPACITY_OVERFLOW);
 
     if (count <= SET_ARRAY_SIZE)
         return SET_ARRAY_SIZE;
 
-    return 1 << (mozilla::FloorLog2(count) + 2);
+    return 1u << (mozilla::FloorLog2(count) + 2);
 }
 
 /* Compute the FNV hash for the low 32 bits of v. */
@@ -869,6 +871,9 @@ HashSetInsertTry(LifoAlloc &alloc, U **&values, unsigned &count, T key)
         }
     }
 
+    if (count >= SET_CAPACITY_OVERFLOW)
+        return nullptr;
+
     count++;
     unsigned newCapacity = HashSetCapacity(count);