Bug 1251919 - Nuke Debugger wrappers on failure. (r=shu) a=ritu
authorEric Faust <efaustbmo@gmail.com>
Tue, 03 May 2016 03:55:14 -0700
changeset 332784 ed90d8e648124ef21e0c616fa1c8cc379b91da90
parent 332783 afe313884a55cfa3f3029fc7cee3e0e5d6ff37f6
child 332785 2d2c08b645dfc3d6c35e467c75c9c42eb9792903
push id6048
push userkmoir@mozilla.com
push dateMon, 06 Jun 2016 19:02:08 +0000
treeherdermozilla-beta@46d72a56c57d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersshu, ritu
bugs1251919
milestone48.0a2
Bug 1251919 - Nuke Debugger wrappers on failure. (r=shu) a=ritu
js/src/jit-test/tests/debug/bug1251919.js
js/src/vm/Debugger.cpp
js/src/vm/Debugger.h
new file mode 100644
--- /dev/null
+++ b/js/src/jit-test/tests/debug/bug1251919.js
@@ -0,0 +1,13 @@
+// |jit-test| error: out of memory
+
+if (!('oomTest' in this))
+    throw new Error("out of memory");
+
+// jsfunfuzz-generated
+fullcompartmentchecks(true);
+// Adapted from randomly chosen test: js/src/jit-test/tests/debug/bug-1248162.js
+var dbg = new Debugger;
+dbg.onNewGlobalObject = function() {};
+oomTest(function() {
+    newGlobal();
+})
--- a/js/src/vm/Debugger.cpp
+++ b/js/src/vm/Debugger.cpp
@@ -393,16 +393,24 @@ class MOZ_RAII js::LeaveDebuggeeNoExecut
 /* static */ bool
 Debugger::slowPathCheckNoExecute(JSContext* cx, HandleScript script)
 {
     MOZ_ASSERT(cx->compartment()->isDebuggee());
     MOZ_ASSERT(cx->runtime()->noExecuteDebuggerTop);
     return EnterDebuggeeNoExecute::reportIfFoundInStack(cx, script);
 }
 
+static inline void
+NukeDebuggerWrapper(NativeObject *wrapper)
+{
+    // In some OOM failure cases, we need to destroy the edge to the referent,
+    // to avoid trying to trace it during untimely collections.
+    wrapper->setPrivate(nullptr);
+}
+
 
 /*** Breakpoints *********************************************************************************/
 
 BreakpointSite::BreakpointSite(JSScript* script, jsbytecode* pc)
   : script(script), pc(pc), enabledCount(0)
 {
     MOZ_ASSERT(!script->hasBreakpointsAt(pc));
     JS_INIT_CLIST(&breakpoints);
@@ -929,24 +937,29 @@ Debugger::wrapEnvironment(JSContext* cx,
         /* Create a new Debugger.Environment for env. */
         RootedObject proto(cx, &object->getReservedSlot(JSSLOT_DEBUG_ENV_PROTO).toObject());
         envobj = NewNativeObjectWithGivenProto(cx, &DebuggerEnv_class, proto,
                                                TenuredObject);
         if (!envobj)
             return false;
         envobj->setPrivateGCThing(env);
         envobj->setReservedSlot(JSSLOT_DEBUGENV_OWNER, ObjectValue(*object));
-        if (!p.add(cx, environments, env, envobj))
-            return false;
+
+        if (!p.add(cx, environments, env, envobj)) {
+            NukeDebuggerWrapper(envobj);
+            return false;
+        }
 
         CrossCompartmentKey key(CrossCompartmentKey::DebuggerEnvironment, object, env);
         if (!object->compartment()->putWrapper(cx, key, ObjectValue(*envobj))) {
+            NukeDebuggerWrapper(envobj);
             environments.remove(env);
             return false;
         }
+
     }
     rval.setObject(*envobj);
     return true;
 }
 
 bool
 Debugger::wrapDebuggeeValue(JSContext* cx, MutableHandleValue vp)
 {
@@ -971,22 +984,25 @@ Debugger::wrapDebuggeeValue(JSContext* c
             NativeObject* dobj =
                 NewNativeObjectWithGivenProto(cx, &DebuggerObject_class, proto,
                                               TenuredObject);
             if (!dobj)
                 return false;
             dobj->setPrivateGCThing(obj);
             dobj->setReservedSlot(JSSLOT_DEBUGOBJECT_OWNER, ObjectValue(*object));
 
-            if (!p.add(cx, objects, obj, dobj))
+            if (!p.add(cx, objects, obj, dobj)) {
+                NukeDebuggerWrapper(dobj);
                 return false;
+            }
 
             if (obj->compartment() != object->compartment()) {
                 CrossCompartmentKey key(CrossCompartmentKey::DebuggerObject, object, obj);
                 if (!object->compartment()->putWrapper(cx, key, ObjectValue(*dobj))) {
+                    NukeDebuggerWrapper(dobj);
                     objects.remove(obj);
                     ReportOutOfMemory(cx);
                     return false;
                 }
             }
 
             vp.setObject(*dobj);
         }
@@ -5023,17 +5039,17 @@ class DebuggerScriptSetPrivateMatcher
     NativeObject* obj_;
   public:
     explicit DebuggerScriptSetPrivateMatcher(NativeObject* obj) : obj_(obj) { }
     using ReturnType = void;
     ReturnType match(HandleScript script) { obj_->setPrivateGCThing(script); }
     ReturnType match(Handle<WasmModuleObject*> module) { obj_->setPrivateGCThing(module); }
 };
 
-JSObject*
+NativeObject*
 Debugger::newDebuggerScript(JSContext* cx, Handle<DebuggerScriptReferent> referent)
 {
     assertSameCompartment(cx, object.get());
 
     RootedObject proto(cx, &object->getReservedSlot(JSSLOT_DEBUG_SCRIPT_PROTO).toObject());
     MOZ_ASSERT(proto);
     NativeObject* scriptobj = NewNativeObjectWithGivenProto(cx, &DebuggerScript_class,
                                                             proto, TenuredObject);
@@ -5053,29 +5069,33 @@ Debugger::wrapVariantReferent(JSContext*
 {
     assertSameCompartment(cx, object);
 
     Handle<Referent> untaggedReferent = referent.template as<Referent>();
     MOZ_ASSERT(cx->compartment() != untaggedReferent->compartment());
 
     DependentAddPtr<Map> p(cx, map, untaggedReferent);
     if (!p) {
-        JSObject* wrapper = newVariantWrapper(cx, referent);
+        NativeObject* wrapper = newVariantWrapper(cx, referent);
         if (!wrapper)
             return nullptr;
 
-        if (!p.add(cx, map, untaggedReferent, wrapper))
+        if (!p.add(cx, map, untaggedReferent, wrapper)) {
+            NukeDebuggerWrapper(wrapper);
             return nullptr;
+        }
 
         CrossCompartmentKey key(keyKind, object, untaggedReferent);
         if (!object->compartment()->putWrapper(cx, key, ObjectValue(*wrapper))) {
+            NukeDebuggerWrapper(wrapper);
             map.remove(untaggedReferent);
             ReportOutOfMemory(cx);
             return nullptr;
         }
+
     }
 
     return p->value();
 }
 
 JSObject*
 Debugger::wrapVariantReferent(JSContext* cx, Handle<DebuggerScriptReferent> referent)
 {
@@ -6295,17 +6315,17 @@ class SetDebuggerSourcePrivateMatcher
     NativeObject* obj_;
   public:
     explicit SetDebuggerSourcePrivateMatcher(NativeObject* obj) : obj_(obj) { }
     using ReturnType = void;
     ReturnType match(HandleScriptSource source) { obj_->setPrivateGCThing(source); }
     ReturnType match(Handle<WasmModuleObject*> module) { obj_->setPrivateGCThing(module); }
 };
 
-JSObject*
+NativeObject*
 Debugger::newDebuggerSource(JSContext* cx, Handle<DebuggerSourceReferent> referent)
 {
     assertSameCompartment(cx, object.get());
 
     RootedObject proto(cx, &object->getReservedSlot(JSSLOT_DEBUG_SOURCE_PROTO).toObject());
     MOZ_ASSERT(proto);
     NativeObject* sourceobj = NewNativeObjectWithGivenProto(cx, &DebuggerSource_class,
                                                             proto, TenuredObject);
--- a/js/src/vm/Debugger.h
+++ b/js/src/vm/Debugger.h
@@ -685,20 +685,20 @@ class Debugger : private mozilla::Linked
                                      FireHookFun fireHook);
 
     JSTrapStatus fireDebuggerStatement(JSContext* cx, MutableHandleValue vp);
     JSTrapStatus fireExceptionUnwind(JSContext* cx, MutableHandleValue vp);
     JSTrapStatus fireEnterFrame(JSContext* cx, AbstractFramePtr frame, MutableHandleValue vp);
     JSTrapStatus fireNewGlobalObject(JSContext* cx, Handle<GlobalObject*> global, MutableHandleValue vp);
     JSTrapStatus firePromiseHook(JSContext* cx, Hook hook, HandleObject promise, MutableHandleValue vp);
 
-    JSObject* newVariantWrapper(JSContext* cx, Handle<DebuggerScriptReferent> referent) {
+    NativeObject* newVariantWrapper(JSContext* cx, Handle<DebuggerScriptReferent> referent) {
         return newDebuggerScript(cx, referent);
     }
-    JSObject* newVariantWrapper(JSContext* cx, Handle<DebuggerSourceReferent> referent) {
+    NativeObject* newVariantWrapper(JSContext* cx, Handle<DebuggerSourceReferent> referent) {
         return newDebuggerSource(cx, referent);
     }
 
     /*
      * Helper function to help wrap Debugger objects whose referents may be
      * variants. Currently Debugger.Script and Debugger.Source referents may
      * be variants.
      *
@@ -710,23 +710,23 @@ class Debugger : private mozilla::Linked
                                   Handle<ReferentVariant> referent);
     JSObject* wrapVariantReferent(JSContext* cx, Handle<DebuggerScriptReferent> referent);
     JSObject* wrapVariantReferent(JSContext* cx, Handle<DebuggerSourceReferent> referent);
 
     /*
      * Allocate and initialize a Debugger.Script instance whose referent is
      * |referent|.
      */
-    JSObject* newDebuggerScript(JSContext* cx, Handle<DebuggerScriptReferent> referent);
+    NativeObject* newDebuggerScript(JSContext* cx, Handle<DebuggerScriptReferent> referent);
 
     /*
      * Allocate and initialize a Debugger.Source instance whose referent is
      * |referent|.
      */
-    JSObject* newDebuggerSource(JSContext* cx, Handle<DebuggerSourceReferent> referent);
+    NativeObject* newDebuggerSource(JSContext* cx, Handle<DebuggerSourceReferent> referent);
 
     /*
      * Receive a "new script" event from the engine. A new script was compiled
      * or deserialized.
      */
     void fireNewScript(JSContext* cx, Handle<DebuggerScriptReferent> scriptReferent);
 
     /*