Bug 1240546 - Handle OOM in updateObservesAllExecutionOnDebuggees. (r=jimb)
authorShu-yu Guo <shu@rfrn.org>
Thu, 21 Jan 2016 18:17:38 -0800
changeset 281133 4e9d8d7b66685fa7c86a21be6d337219ad84fa90
parent 281132 755c6f29cbae1a3e4a8a27bca8d046224c11a275
child 281134 96fd3c4b8bfdf6e78e1cd09db58060a9a03aec58
push id70673
push usershu@rfrn.org
push dateFri, 22 Jan 2016 02:14:03 +0000
treeherdermozilla-inbound@96fd3c4b8bfd [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjimb
bugs1240546
milestone46.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1240546 - Handle OOM in updateObservesAllExecutionOnDebuggees. (r=jimb)
js/src/jit-test/tests/debug/bug1240546.js
js/src/vm/Debugger.cpp
new file mode 100644
--- /dev/null
+++ b/js/src/jit-test/tests/debug/bug1240546.js
@@ -0,0 +1,12 @@
+// |jit-test| allow-oom
+
+if (!('oomAfterAllocations' in this))
+  quit();
+
+var g = newGlobal();
+g.debuggeeGlobal = this;
+g.eval("(" + function() {
+    oomAfterAllocations(100);
+    var dbg = Debugger(debuggeeGlobal);
+    dbg.onEnterFrame = function(frame) {}
+} + ")()");
--- a/js/src/vm/Debugger.cpp
+++ b/js/src/vm/Debugger.cpp
@@ -2288,21 +2288,26 @@ Debugger::updateObservesAllExecutionOnDe
 
         if (comp->debuggerObservesAllExecution() == observing)
             continue;
 
         // It's expensive to eagerly invalidate and recompile a compartment,
         // so add the compartment to the set only if we are observing.
         if (observing && !obs.add(comp))
             return false;
-
-        comp->updateDebuggerObservesAllExecution();
-    }
-
-    return updateExecutionObservability(cx, obs, observing);
+    }
+
+    if (!updateExecutionObservability(cx, obs, observing))
+        return false;
+
+    typedef ExecutionObservableCompartments::CompartmentRange CompartmentRange;
+    for (CompartmentRange r = obs.compartments()->all(); !r.empty(); r.popFront())
+        r.front()->updateDebuggerObservesAllExecution();
+
+    return true;
 }
 
 bool
 Debugger::updateObservesCoverageOnDebuggees(JSContext* cx, IsObserving observing)
 {
     ExecutionObservableCompartments obs(cx);
     if (!obs.init())
         return false;
@@ -2852,20 +2857,24 @@ Debugger::setHookImpl(JSContext* cx, Cal
         return false;
     if (args[0].isObject()) {
         if (!args[0].toObject().isCallable())
             return ReportIsNotFunction(cx, args[0], args.length() - 1);
     } else if (!args[0].isUndefined()) {
         JS_ReportErrorNumber(cx, GetErrorMessage, nullptr, JSMSG_NOT_CALLABLE_OR_UNDEFINED);
         return false;
     }
-    dbg.object->setReservedSlot(JSSLOT_DEBUG_HOOK_START + which, args[0]);
+    uint32_t slot = JSSLOT_DEBUG_HOOK_START + which;
+    RootedValue oldHook(cx, dbg.object->getReservedSlot(slot));
+    dbg.object->setReservedSlot(slot, args[0]);
     if (hookObservesAllExecution(which)) {
-        if (!dbg.updateObservesAllExecutionOnDebuggees(cx, dbg.observesAllExecution()))
-            return false;
+        if (!dbg.updateObservesAllExecutionOnDebuggees(cx, dbg.observesAllExecution())) {
+            dbg.object->setReservedSlot(slot, oldHook);
+            return false;
+        }
     }
     args.rval().setUndefined();
     return true;
 }
 
 /* static */ bool
 Debugger::getOnDebuggerStatement(JSContext* cx, unsigned argc, Value* vp)
 {