Detect OOM condition and flush code cache when destroying the recorder and don't enter endAssembly when OOM (475821, r=danderson).
authorAndreas Gal <gal@mozilla.com>
Wed, 28 Jan 2009 15:12:31 -0800
changeset 24486 9d9412b90552177416507e515b75ebbc2fdac0f4
parent 24385 0c727ce8bb5f48c7045cec494f5b5ee4ac6df6af
child 24487 7246c4dcf99709f0fe1e0aefbf812e701f1961ad
push id5074
push userrsayre@mozilla.com
push dateSat, 31 Jan 2009 19:45:42 +0000
treeherdermozilla-central@f1cade532f6f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdanderson
bugs475821
milestone1.9.2a1pre
Detect OOM condition and flush code cache when destroying the recorder and don't enter endAssembly when OOM (475821, r=danderson).
js/src/jstracer.cpp
js/src/nanojit/Assembler.cpp
--- a/js/src/jstracer.cpp
+++ b/js/src/jstracer.cpp
@@ -2955,24 +2955,34 @@ nanojit::LirNameMap::formatGuard(LIns *i
 #endif
 
 void
 nanojit::Fragment::onDestroy()
 {
     delete (TreeInfo *)vmprivate;
 }
 
-void
+bool
 js_DeleteRecorder(JSContext* cx)
 {
     JSTraceMonitor* tm = &JS_TRACE_MONITOR(cx);
 
     /* Aborting and completing a trace end up here. */
     delete tm->recorder;
     tm->recorder = NULL;
+
+    /* 
+     * If we ran out of memory, flush the code cache.
+     */
+    if (JS_TRACE_MONITOR(cx).fragmento->assm()->error() == OutOMem) {
+        js_FlushJITCache(cx);
+        return false;
+    }
+
+    return true;
 }
 
 /**
  * Checks whether the shape of the global object has changed.
  */
 static inline bool
 js_CheckGlobalObjectShape(JSContext* cx, JSTraceMonitor* tm, JSObject* globalObj)
 {
@@ -3484,28 +3494,29 @@ js_CloseLoop(JSContext* cx)
     JSTraceMonitor* tm = &JS_TRACE_MONITOR(cx);
     Fragmento* fragmento = tm->fragmento;
     TraceRecorder* r = tm->recorder;
     JS_ASSERT(fragmento && r);
     bool walkedOutOfLoop = r->walkedOutOfLoop();
     
     if (fragmento->assm()->error()) {
         js_AbortRecording(cx, "Error during recording");
-
-        /* If we ran out of memory, flush the code cache and abort. */
-        if (fragmento->assm()->error() == OutOMem)
-            js_FlushJITCache(cx);
         return false;
     }
 
     bool demote = false;
     Fragment* f = r->getFragment();
     r->closeLoop(tm, demote);
-    js_DeleteRecorder(cx);
-    
+
+    /* 
+     * If js_DeleteRecorder flushed the code cache, we can't rely on f any more.
+     */
+    if (!js_DeleteRecorder(cx))
+        return false;
+
     /*
      * If we just walked out of a thin loop, we can't immediately start the 
      * compiler again here since we didn't return to the loop header.
      */
     if (demote && !walkedOutOfLoop)
         return js_RecordTree(cx, tm, f, NULL);
     return false;
 }
@@ -4278,18 +4289,27 @@ js_AbortRecording(JSContext* cx, const c
         return;
     }
     JS_ASSERT(!f->vmprivate);
     js_BlacklistPC(tm, f);
     Fragment* outer = tm->recorder->getOuterToBlacklist();
     /* Give outer two chances to stabilize before we start blacklisting. */
     if (outer != NULL && outer->recordAttempts >= 2)
         js_BlacklistPC(tm, outer);
-    js_DeleteRecorder(cx);
-    /* If this is the primary trace and we didn't succeed compiling, trash the TreeInfo object. */
+
+    /* 
+     * If js_DeleteRecorder flushed the code cache, we can't rely on f any more.
+     */
+    if (!js_DeleteRecorder(cx))
+        return;
+
+    /*
+     * If this is the primary trace and we didn't succeed compiling, trash the
+     * TreeInfo object.
+     */
     if (!f->code() && (f->root == f)) 
         js_TrashTree(cx, f);
 }
 
 #if defined NANOJIT_IA32
 static bool
 js_CheckForSSE2()
 {
--- a/js/src/nanojit/Assembler.cpp
+++ b/js/src/nanojit/Assembler.cpp
@@ -849,16 +849,21 @@ namespace nanojit
         }
 		else {
 			_nIns = _startingIns;  // in case of failure reset nIns ready for the next assembly run
 		}
 	}
 
 	void Assembler::endAssembly(Fragment* frag, NInsList& loopJumps)
 	{
+		// don't try to patch code if we are in an error state since we might have partially 
+		// overwritten the code cache already
+		if (error())
+			return;
+
 	    NIns* SOT = 0;
 	    if (frag->isRoot()) {
 	        SOT = frag->loopEntry;
             verbose_only( verbose_outputf("        %p:",_nIns); )
 	    } else {
 	        SOT = frag->root->fragEntry;
 	    }
         AvmAssert(SOT);