Bug 605752: don't crash on OOM inside ExecutablePool, r=dvander, a=beta8+
authorDavid Mandelin <dmandelin@mozilla.com>
Fri, 12 Nov 2010 18:17:21 -0800
changeset 57437 95e9c2e8708d3e089183a751fe2c7e2eb11d0065
parent 57436 255ec74f08b40cef03d88ae50e9e7f3d2364d089
child 57438 a871bd60be99e7e360306bf1c0b7f4d4394bb251
push id1
push usershaver@mozilla.com
push dateTue, 04 Jan 2011 17:58:04 +0000
reviewersdvander, beta8
bugs605752
milestone2.0b8pre
Bug 605752: don't crash on OOM inside ExecutablePool, r=dvander, a=beta8+
js/src/assembler/jit/ExecutableAllocator.h
js/src/assembler/jit/ExecutableAllocatorPosix.cpp
js/src/assembler/jit/ExecutableAllocatorSymbian.cpp
js/src/assembler/jit/ExecutableAllocatorWin.cpp
js/src/jsapi.cpp
js/src/methodjit/MethodJIT.cpp
--- a/js/src/assembler/jit/ExecutableAllocator.h
+++ b/js/src/assembler/jit/ExecutableAllocator.h
@@ -77,20 +77,22 @@ extern "C" __declspec(dllimport) void Ca
 namespace JSC {
 
 // Something included via windows.h defines a macro with this name,
 // which causes the function below to fail to compile.
 #ifdef _MSC_VER
 # undef max
 #endif
 
+const size_t OVERSIZE_ALLOCATION = size_t(-1);
+
 inline size_t roundUpAllocationSize(size_t request, size_t granularity)
 {
     if ((std::numeric_limits<size_t>::max() - granularity) <= request)
-        CRASH(); // Allocation is too large
+        return OVERSIZE_ALLOCATION;
     
     // Round up to next page boundary
     size_t size = request + (granularity - 1);
     size = size & ~(granularity - 1);
     JS_ASSERT(size >= request);
     return size;
 }
 
@@ -125,26 +127,33 @@ public:
 	  JS_ASSERT(m_refCount != 0);
 	  if (JS_ATOMIC_DECREMENT(&m_refCount) == 0) 
 	      delete this; 
       }
 
     //static PassRefPtr<ExecutablePool> create(size_t n)
     static ExecutablePool* create(size_t n)
     {
-        return new ExecutablePool(n);
+        ExecutablePool *pool = new ExecutablePool(n);
+        if (!pool->m_freePtr) {
+            delete pool;
+            return NULL;
+        }
+        return pool;
     }
 
     void* alloc(size_t n)
     {
         JS_ASSERT(m_freePtr <= m_end);
 
         // Round 'n' up to a multiple of word size; if all allocations are of
         // word sized quantities, then all subsequent allocations will be aligned.
         n = roundUpAllocationSize(n, sizeof(void*));
+        if (n == OVERSIZE_ALLOCATION)
+            return NULL;
 
         if (static_cast<ptrdiff_t>(n) < (m_end - m_freePtr)) {
             void* result = m_freePtr;
             m_freePtr += n;
             return result;
         }
 
         // Insufficient space to allocate in the existing pool
@@ -157,38 +166,51 @@ public:
         Allocation* end = m_pools.end();
         for (Allocation* ptr = m_pools.begin(); ptr != end; ++ptr)
             ExecutablePool::systemRelease(*ptr);
     }
 
     size_t available() const { return (m_pools.length() > 1) ? 0 : m_end - m_freePtr; }
 
 private:
+    // On OOM, this will return an Allocation where pages is NULL.
     static Allocation systemAlloc(size_t n);
     static void systemRelease(const Allocation& alloc);
 
     ExecutablePool(size_t n);
 
     void* poolAllocate(size_t n);
 
     char* m_freePtr;
     char* m_end;
     AllocationList m_pools;
 };
 
 class ExecutableAllocator {
     enum ProtectionSeting { Writable, Executable };
 
+    // Initialization can fail so we use a create method instead.
+    ExecutableAllocator() {}
 public:
     static size_t pageSize;
-    ExecutableAllocator()
+
+    // Returns NULL on OOM.
+    static ExecutableAllocator *create()
     {
+        ExecutableAllocator *allocator = new ExecutableAllocator();
+        if (!allocator)
+            return allocator;
+
         if (!pageSize)
             intializePageSize();
-        m_smallAllocationPool = ExecutablePool::create(JIT_ALLOCATOR_LARGE_ALLOC_SIZE);
+        allocator->m_smallAllocationPool = ExecutablePool::create(JIT_ALLOCATOR_LARGE_ALLOC_SIZE);
+        if (!allocator->m_smallAllocationPool) {
+            delete allocator;
+            return NULL;
+        }
     }
 
     ~ExecutableAllocator() { delete m_smallAllocationPool; }
 
     // poolForSize returns reference-counted objects. The caller owns a reference
     // to the object; i.e., poolForSize increments the count before returning the
     // object.
 
@@ -201,16 +223,18 @@ public:
 	}
 
         // If the request is large, we just provide a unshared allocator
         if (n > JIT_ALLOCATOR_LARGE_ALLOC_SIZE)
             return ExecutablePool::create(n);
 
         // Create a new allocator
         ExecutablePool* pool = ExecutablePool::create(JIT_ALLOCATOR_LARGE_ALLOC_SIZE);
+        if (!pool)
+            return NULL;
   	    // At this point, local |pool| is the owner.
 
         // If the new allocator will result in more free space than in
         // the current small allocator, then we will use it instead
         if ((pool->available() - n) > m_smallAllocationPool->available()) {
 	        m_smallAllocationPool->release();
             m_smallAllocationPool = pool;
 	        pool->addRef();
@@ -325,34 +349,48 @@ private:
 #if ENABLE_ASSEMBLER_WX_EXCLUSIVE
     static void reprotectRegion(void*, size_t, ProtectionSeting);
 #endif
 
     ExecutablePool* m_smallAllocationPool;
     static void intializePageSize();
 };
 
+// This constructor can fail due to OOM. If it does, m_freePtr will be
+// set to NULL. 
 inline ExecutablePool::ExecutablePool(size_t n) : m_refCount(1)
 {
     size_t allocSize = roundUpAllocationSize(n, JIT_ALLOCATOR_PAGE_SIZE);
+    if (allocSize == OVERSIZE_ALLOCATION) {
+        m_freePtr = NULL;
+        return;
+    }
     Allocation mem = systemAlloc(allocSize);
-    m_pools.append(mem);
+    if (!mem.pages) {
+        m_freePtr = NULL;
+        return;
+    }
+    if (!m_pools.append(mem)) {
+        systemRelease(mem);
+        m_freePtr = NULL;
+        return;
+    }
     m_freePtr = mem.pages;
-    if (!m_freePtr)
-        CRASH(); // Failed to allocate
     m_end = m_freePtr + allocSize;
 }
 
 inline void* ExecutablePool::poolAllocate(size_t n)
 {
     size_t allocSize = roundUpAllocationSize(n, JIT_ALLOCATOR_PAGE_SIZE);
+    if (allocSize == OVERSIZE_ALLOCATION)
+        return NULL;
     
     Allocation result = systemAlloc(allocSize);
     if (!result.pages)
-        CRASH(); // Failed to allocate
+        return NULL;
     
     JS_ASSERT(m_end >= m_freePtr);
     if ((allocSize - n) > static_cast<size_t>(m_end - m_freePtr)) {
         // Replace allocation pool
         m_freePtr = result.pages + n;
         m_end = result.pages + allocSize;
     }
 
--- a/js/src/assembler/jit/ExecutableAllocatorPosix.cpp
+++ b/js/src/assembler/jit/ExecutableAllocatorPosix.cpp
@@ -37,17 +37,17 @@ void ExecutableAllocator::intializePageS
 {
     ExecutableAllocator::pageSize = getpagesize();
 }
 
 ExecutablePool::Allocation ExecutablePool::systemAlloc(size_t n)
 {
     void* allocation = mmap(NULL, n, INITIAL_PROTECTION_FLAGS, MAP_PRIVATE | MAP_ANON, VM_TAG_FOR_EXECUTABLEALLOCATOR_MEMORY, 0);
     if (allocation == MAP_FAILED)
-        CRASH();
+        allocation = NULL;
     ExecutablePool::Allocation alloc = { reinterpret_cast<char*>(allocation), n };
     return alloc;
 }
 
 void ExecutablePool::systemRelease(const ExecutablePool::Allocation& alloc)
 { 
     int result = munmap(alloc.pages, alloc.size);
     ASSERT_UNUSED(result, !result);
--- a/js/src/assembler/jit/ExecutableAllocatorSymbian.cpp
+++ b/js/src/assembler/jit/ExecutableAllocatorSymbian.cpp
@@ -49,18 +49,16 @@ void ExecutableAllocator::intializePageS
 
 ExecutablePool::Allocation ExecutablePool::systemAlloc(size_t n)
 {
     RChunk* codeChunk = new RChunk();
 
     TInt errorCode = codeChunk->CreateLocalCode(n, n);
 
     char* allocation = reinterpret_cast<char*>(codeChunk->Base());
-    if (!allocation)
-        CRASH();
     ExecutablePool::Allocation alloc = { allocation, n, codeChunk };
     return alloc;
 }
 
 void ExecutablePool::systemRelease(const ExecutablePool::Allocation& alloc)
 { 
     alloc.chunk->Close();
     delete alloc.chunk;
--- a/js/src/assembler/jit/ExecutableAllocatorWin.cpp
+++ b/js/src/assembler/jit/ExecutableAllocatorWin.cpp
@@ -36,19 +36,17 @@ void ExecutableAllocator::intializePageS
 {
     SYSTEM_INFO system_info;
     GetSystemInfo(&system_info);
     ExecutableAllocator::pageSize = system_info.dwPageSize;
 }
 
 ExecutablePool::Allocation ExecutablePool::systemAlloc(size_t n)
 {
-    void* allocation = VirtualAlloc(0, n, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE);
-    if (!allocation)
-        CRASH();
+    void *allocation = VirtualAlloc(0, n, MEM_COMMIT | MEM_RESERVE, PAGE_EXECUTE_READWRITE);
     ExecutablePool::Allocation alloc = {reinterpret_cast<char*>(allocation), n};
     return alloc;
 }
 
 void ExecutablePool::systemRelease(const ExecutablePool::Allocation& alloc)
 { 
     VirtualFree(alloc.pages, 0, MEM_RELEASE); 
 }
--- a/js/src/jsapi.cpp
+++ b/js/src/jsapi.cpp
@@ -623,17 +623,17 @@ JSRuntime::init(uint32 maxbytes)
         !compartments.append(defaultCompartment)) {
         return false;
     }
 
     if (!js_InitGC(this, maxbytes) || !js_InitAtomState(this))
         return false;
 
 #if ENABLE_YARR_JIT
-    regExpAllocator = new JSC::ExecutableAllocator();
+    regExpAllocator = JSC::ExecutableAllocator::create();
     if (!regExpAllocator)
         return false;
 #endif
 
     deflatedStringCache = new js::DeflatedStringCache();
     if (!deflatedStringCache || !deflatedStringCache->init())
         return false;
 
--- a/js/src/methodjit/MethodJIT.cpp
+++ b/js/src/methodjit/MethodJIT.cpp
@@ -677,17 +677,17 @@ JS_STATIC_ASSERT(JSVAL_PAYLOAD_MASK == 0
 #  error "Unsupported CPU!"
 #endif
 
 #endif                   /* _MSC_VER */
 
 bool
 JaegerCompartment::Initialize()
 {
-    execAlloc = new JSC::ExecutableAllocator();
+    execAlloc = JSC::ExecutableAllocator::create();
     if (!execAlloc)
         return false;
     
     TrampolineCompiler tc(execAlloc, &trampolines);
     if (!tc.compile()) {
         delete execAlloc;
         return false;
     }