Bug 1109328 - Fix an OOM case when compiling debug instrumentation in Baseline. (r=jandem)
authorShu-yu Guo <shu@rfrn.org>
Mon, 15 Dec 2014 18:21:08 -0800
changeset 219861 049230caef34282c6307300d0cb41d100fed71c1
parent 219860 877e91964ea927db0fd47afe60b20c5c2da821ce
child 219862 f10fd10b7e27ffc969aa327fbd4839b72746552d
push id10419
push usercbook@mozilla.com
push dateTue, 16 Dec 2014 12:45:27 +0000
treeherderfx-team@ec87657146eb [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjandem
bugs1109328
milestone37.0a1
Bug 1109328 - Fix an OOM case when compiling debug instrumentation in Baseline. (r=jandem)
js/src/jit-test/tests/debug/bug1109328.js
js/src/jit/BaselineCompiler.cpp
js/src/vm/Debugger.cpp
new file mode 100644
--- /dev/null
+++ b/js/src/jit-test/tests/debug/bug1109328.js
@@ -0,0 +1,7 @@
+try {
+    gcslice(0)(""());
+} catch (e) {}
+g = newGlobal()
+g.parent = this
+g.eval("Debugger(parent).onExceptionUnwind=(function(){})");
+gcparam("maxBytes", gcparam("gcBytes"));
--- a/js/src/jit/BaselineCompiler.cpp
+++ b/js/src/jit/BaselineCompiler.cpp
@@ -749,16 +749,18 @@ BaselineCompiler::emitDebugTrap()
 {
     MOZ_ASSERT(compileDebugInstrumentation_);
     MOZ_ASSERT(frame.numUnsyncedSlots() == 0);
 
     bool enabled = script->stepModeEnabled() || script->hasBreakpointsAt(pc);
 
     // Emit patchable call to debug trap handler.
     JitCode *handler = cx->runtime()->jitRuntime()->debugTrapHandler(cx);
+    if (!handler)
+        return false;
     mozilla::DebugOnly<CodeOffsetLabel> offset = masm.toggledCall(handler, enabled);
 
 #ifdef DEBUG
     // Patchable call offset has to match the pc mapping offset.
     PCMappingEntry &entry = pcMappingEntries_.back();
     MOZ_ASSERT((&offset)->offset() == entry.nativeOffset);
 #endif
 
--- a/js/src/vm/Debugger.cpp
+++ b/js/src/vm/Debugger.cpp
@@ -438,41 +438,41 @@ Debugger::getScriptFrameWithIter(JSConte
                                  const ScriptFrameIter *maybeIter, MutableHandleValue vp)
 {
     MOZ_ASSERT_IF(maybeIter, maybeIter->abstractFramePtr() == frame);
 
     FrameMap::AddPtr p = frames.lookupForAdd(frame);
     if (!p) {
         /* Create and populate the Debugger.Frame object. */
         JSObject *proto = &object->getReservedSlot(JSSLOT_DEBUG_FRAME_PROTO).toObject();
-        NativeObject *frameobj =
-            NewNativeObjectWithGivenProto(cx, &DebuggerFrame_class, proto, nullptr);
+        RootedNativeObject frameobj(cx, NewNativeObjectWithGivenProto(cx, &DebuggerFrame_class,
+                                                                      proto, nullptr));
         if (!frameobj)
             return false;
 
         // Eagerly copy ScriptFrameIter data if we've already walked the
         // stack.
         if (maybeIter) {
             AbstractFramePtr data = maybeIter->copyDataAsAbstractFramePtr();
             if (!data)
                 return false;
             frameobj->setPrivate(data.raw());
         } else {
             frameobj->setPrivate(frame.raw());
         }
 
         frameobj->setReservedSlot(JSSLOT_DEBUGFRAME_OWNER, ObjectValue(*object));
 
+        if (!ensureExecutionObservabilityOfFrame(cx, frame))
+            return false;
+
         if (!frames.add(p, frame, frameobj)) {
             js_ReportOutOfMemory(cx);
             return false;
         }
-
-        if (!ensureExecutionObservabilityOfFrame(cx, frame))
-            return false;
     }
     vp.setObject(*p->value());
     return true;
 }
 
 /* static */ bool
 Debugger::hasLiveHook(GlobalObject *global, Hook which)
 {