Bug 640265 - Don't GC during OOM reporting (r=dmandelin)
authorBill McCloskey <wmccloskey@mozilla.com>
Tue, 17 May 2011 11:23:31 -0700
changeset 69929 183f04be1d45
parent 69928 1c4b22465a5d
child 69930 5186a7fe721b
push id20142
push usercleary@mozilla.com
push date2011-05-23 07:31 +0000
Treeherderresults
reviewersdmandelin
bugs640265
milestone6.0a1
Bug 640265 - Don't GC during OOM reporting (r=dmandelin)
js/src/jscntxt.cpp
js/src/jscntxt.h
js/src/jsgc.cpp
js/src/jstl.h
--- a/js/src/jscntxt.cpp
+++ b/js/src/jscntxt.cpp
@@ -815,18 +815,20 @@ js_ReportOutOfMemory(JSContext *cx)
     if (onError) {
         JSDebugErrorHook hook = cx->debugHooks->debugErrorHook;
         if (hook &&
             !hook(cx, msg, &report, cx->debugHooks->debugErrorHookData)) {
             onError = NULL;
         }
     }
 
-    if (onError)
+    if (onError) {
+        AutoScopedAssign<bool> ss(&cx->runtime->inOOMReport, true);
         onError(cx, msg, &report);
+    }
 }
 
 void
 js_ReportOutOfScriptQuota(JSContext *maybecx)
 {
     if (maybecx)
         JS_ReportErrorNumber(maybecx, js_GetErrorMessage, NULL, JSMSG_SCRIPT_STACK_QUOTA);
 }
--- a/js/src/jscntxt.h
+++ b/js/src/jscntxt.h
@@ -700,37 +700,44 @@ struct JSRuntime {
     JSWrapObjectCallback wrapObjectCallback;
     JSPreWrapCallback    preWrapObjectCallback;
 
 #ifdef JS_METHODJIT
     /* This measures the size of JITScripts, native maps and IC structs. */
     size_t               mjitDataSize;
 #endif
 
+    /*
+     * To ensure that cx->malloc does not cause a GC, we set this flag during
+     * OOM reporting (in js_ReportOutOfMemory). If a GC is requested while
+     * reporting the OOM, we ignore it.
+     */
+    bool                 inOOMReport;
+
     JSRuntime();
     ~JSRuntime();
 
     bool init(uint32 maxbytes);
 
     void setGCLastBytes(size_t lastBytes);
     void reduceGCTriggerBytes(uint32 amount);
 
     /*
      * Call the system malloc while checking for GC memory pressure and
-     * reporting OOM error when cx is not null.
+     * reporting OOM error when cx is not null. We will not GC from here.
      */
     void* malloc_(size_t bytes, JSContext *cx = NULL) {
         updateMallocCounter(bytes);
         void *p = ::js_malloc(bytes);
         return JS_LIKELY(!!p) ? p : onOutOfMemory(NULL, bytes, cx);
     }
 
     /*
      * Call the system calloc while checking for GC memory pressure and
-     * reporting OOM error when cx is not null.
+     * reporting OOM error when cx is not null. We will not GC from here.
      */
     void* calloc_(size_t bytes, JSContext *cx = NULL) {
         updateMallocCounter(bytes);
         void *p = ::js_calloc(bytes);
         return JS_LIKELY(!!p) ? p : onOutOfMemory(reinterpret_cast<void *>(1), bytes, cx);
     }
 
     void* realloc_(void* p, size_t oldBytes, size_t newBytes, JSContext *cx = NULL) {
--- a/js/src/jsgc.cpp
+++ b/js/src/jsgc.cpp
@@ -2678,16 +2678,20 @@ GCCycle(JSContext *cx, JSCompartment *co
         (*c)->setGCLastBytes((*c)->gcBytes);
 }
 
 void
 js_GC(JSContext *cx, JSCompartment *comp, JSGCInvocationKind gckind)
 {
     JSRuntime *rt = cx->runtime;
 
+    /* Don't GC while reporting an OOM. */
+    if (rt->inOOMReport)
+        return;
+
     /*
      * Don't collect garbage if the runtime isn't up, and cx is not the last
      * context in the runtime.  The last context must force a GC, and nothing
      * should suppress that final collection or there may be shutdown leaks,
      * or runtime bloat until the next context is created.
      */
     if (rt->state != JSRTS_UP && gckind != GC_LAST_CONTEXT)
         return;
--- a/js/src/jstl.h
+++ b/js/src/jstl.h
@@ -498,11 +498,30 @@ class RangeCheckedPointer
 template <class T, class U>
 JS_ALWAYS_INLINE T &
 ImplicitCast(U &u)
 {
     T &t = u;
     return t;
 }
 
+template<typename T>
+class AutoScopedAssign
+{
+  private:
+    JS_DECL_USE_GUARD_OBJECT_NOTIFIER
+    T *addr;
+    T old;
+
+  public:
+    AutoScopedAssign(T *addr, const T &value JS_GUARD_OBJECT_NOTIFIER_PARAM)
+        : addr(addr), old(*addr)
+    {
+        JS_GUARD_OBJECT_NOTIFIER_INIT;
+        *addr = value;
+    }
+
+    ~AutoScopedAssign() { *addr = old; }
+};
+
 } /* namespace js */
 
 #endif /* jstl_h_ */