Bug 937132 - SpiderMonkey: Check for overflows in LifoAlloc. r=luke, a=bajaj
authorDan Gohman <sunfish@google.com>
Mon, 23 Dec 2013 09:09:05 -0500
changeset 174593 58f46efec92edaa93ede656502f8bbeae34e4d45
parent 174592 6369fcd317d4128d779fe305daa3c822dc2d627c
child 174594 cf7dbb8a1a1d4f039768b97348609237dcbca9cd
push id3224
push userlsblakk@mozilla.com
push dateTue, 04 Feb 2014 01:06:49 +0000
treeherdermozilla-beta@60c04d0987f1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersluke, bajaj
bugs937132
milestone28.0a2
Bug 937132 - SpiderMonkey: Check for overflows in LifoAlloc. r=luke, a=bajaj
js/src/ds/LifoAlloc.h
js/src/jit/FixedList.h
js/src/jit/MIRGenerator.h
js/src/jit/MIRGraph.h
js/src/jsinferinlines.h
--- a/js/src/ds/LifoAlloc.h
+++ b/js/src/ds/LifoAlloc.h
@@ -300,27 +300,26 @@ class LifoAlloc
             return false;
         if (latestBefore)
             latest = latestBefore;
         return true;
     }
 
     template <typename T>
     T *newArray(size_t count) {
-        void *mem = alloc(sizeof(T) * count);
-        if (!mem)
-            return nullptr;
         JS_STATIC_ASSERT(mozilla::IsPod<T>::value);
-        return (T *) mem;
+        return newArrayUninitialized<T>(count);
     }
 
     // Create an array with uninitialized elements of type |T|.
     // The caller is responsible for initialization.
     template <typename T>
     T *newArrayUninitialized(size_t count) {
+        if (count & mozilla::tl::MulOverflowMask<sizeof(T)>::value)
+            return nullptr;
         return static_cast<T *>(alloc(sizeof(T) * count));
     }
 
     class Mark {
         BumpChunk *chunk;
         void *markInChunk;
         friend class LifoAlloc;
         Mark(BumpChunk *chunk, void *markInChunk) : chunk(chunk), markInChunk(markInChunk) {}
--- a/js/src/jit/FixedList.h
+++ b/js/src/jit/FixedList.h
@@ -23,39 +23,46 @@ class FixedList
     T *list_;
 
   private:
     FixedList(const FixedList&); // no copy definition.
     void operator= (const FixedList*); // no assignment definition.
 
   public:
     FixedList()
-      : length_(0)
+      : length_(0), list_(nullptr)
     { }
 
     // Dynamic memory allocation requires the ability to report failure.
     bool init(TempAllocator &alloc, size_t length) {
         length_ = length;
         if (length == 0)
             return true;
 
+        if (length & mozilla::tl::MulOverflowMask<sizeof(T)>::value)
+            return false;
         list_ = (T *)alloc.allocate(length * sizeof(T));
         return list_ != nullptr;
     }
 
     size_t length() const {
         return length_;
     }
 
     void shrink(size_t num) {
         JS_ASSERT(num < length_);
         length_ -= num;
     }
 
     bool growBy(TempAllocator &alloc, size_t num) {
+        size_t newlength = length_ + num;
+        if (newlength < length_)
+            return false;
+        if (newlength & mozilla::tl::MulOverflowMask<sizeof(T)>::value)
+            return false;
         T *list = (T *)alloc.allocate((length_ + num) * sizeof(T));
         if (!list)
             return false;
 
         for (size_t i = 0; i < length_; i++)
             list[i] = list_[i];
 
         length_ += num;
--- a/js/src/jit/MIRGenerator.h
+++ b/js/src/jit/MIRGenerator.h
@@ -48,16 +48,18 @@ class MIRGenerator
         return GetIonContext()->runtime->jitRuntime();
     }
     CompileInfo &info() {
         return *info_;
     }
 
     template <typename T>
     T * allocate(size_t count = 1) {
+        if (count & mozilla::tl::MulOverflowMask<sizeof(T)>::value)
+            return nullptr;
         return reinterpret_cast<T *>(alloc().allocate(sizeof(T) * count));
     }
 
     // Set an error state and prints a message. Returns false so errors can be
     // propagated up.
     bool abort(const char *message, ...);
     bool abortFmt(const char *message, va_list ap);
 
--- a/js/src/jit/MIRGraph.h
+++ b/js/src/jit/MIRGraph.h
@@ -559,21 +559,16 @@ class MIRGraph
         numBlocks_(0),
         hasTryBlock_(false)
     { }
 
     TempAllocator &alloc() const {
         return *alloc_;
     }
 
-    template <typename T>
-    T * allocate(size_t count = 1) {
-        return reinterpret_cast<T *>(alloc_->allocate(sizeof(T) * count));
-    }
-
     void addBlock(MBasicBlock *block);
     void insertBlockAfter(MBasicBlock *at, MBasicBlock *block);
 
     void unmarkBlocks();
 
     void setReturnAccumulator(MIRGraphReturns *accum) {
         returnAccumulator_ = accum;
     }
--- a/js/src/jsinferinlines.h
+++ b/js/src/jsinferinlines.h
@@ -904,27 +904,29 @@ TypeCompartment::resolvePending(JSContex
  * The sets of objects and scripts in a type set grow monotonically, are usually
  * empty, almost always small, and sometimes big.  For empty or singleton sets,
  * the pointer refers directly to the value.  For sets fitting into SET_ARRAY_SIZE,
  * an array of this length is used to store the elements.  For larger sets, a hash
  * table filled to 25%-50% of capacity is used, with collisions resolved by linear
  * probing.  TODO: replace these with jshashtables.
  */
 const unsigned SET_ARRAY_SIZE = 8;
+const unsigned SET_CAPACITY_OVERFLOW = 1u << 30;
 
 /* Get the capacity of a set with the given element count. */
 static inline unsigned
 HashSetCapacity(unsigned count)
 {
     JS_ASSERT(count >= 2);
+    JS_ASSERT(count < SET_CAPACITY_OVERFLOW);
 
     if (count <= SET_ARRAY_SIZE)
         return SET_ARRAY_SIZE;
 
-    return 1 << (mozilla::FloorLog2(count) + 2);
+    return 1u << (mozilla::FloorLog2(count) + 2);
 }
 
 /* Compute the FNV hash for the low 32 bits of v. */
 template <class T, class KEY>
 static inline uint32_t
 HashKey(T v)
 {
     uint32_t nv = KEY::keyBits(v);
@@ -952,16 +954,19 @@ HashSetInsertTry(LifoAlloc &alloc, U **&
     if (!converting) {
         while (values[insertpos] != nullptr) {
             if (KEY::getKey(values[insertpos]) == key)
                 return &values[insertpos];
             insertpos = (insertpos + 1) & (capacity - 1);
         }
     }
 
+    if (count >= SET_CAPACITY_OVERFLOW)
+        return nullptr;
+
     count++;
     unsigned newCapacity = HashSetCapacity(count);
 
     if (newCapacity == capacity) {
         JS_ASSERT(!converting);
         return &values[insertpos];
     }