[INFER] Wait for code to get hot before inlining calls, bug 639099.
authorBrian Hackett <bhackett1024@gmail.com>
Sun, 27 Mar 2011 07:48:03 -0700
changeset 74853 5d8a1a69f53af3a0b2be10c199cf86b84dab7787
parent 74852 1cf3859b5f1e341840fc13f83d5ec8d2df7fca31
child 74854 de52adfa3439f69593872332bab6e21c4ae1bd02
push id2
push userbsmedberg@mozilla.com
push dateFri, 19 Aug 2011 14:38:13 +0000
bugs639099
milestone2.0b13pre
[INFER] Wait for code to get hot before inlining calls, bug 639099.
js/src/jsanalyze.cpp
js/src/jsanalyze.h
js/src/jsdbgapi.cpp
js/src/jsfun.cpp
js/src/jsscript.h
js/src/methodjit/Compiler.cpp
js/src/methodjit/Compiler.h
js/src/methodjit/Logging.cpp
js/src/methodjit/Logging.h
js/src/methodjit/StubCalls.cpp
js/src/methodjit/StubCalls.h
--- a/js/src/jsanalyze.cpp
+++ b/js/src/jsanalyze.cpp
@@ -440,16 +440,22 @@ Script::analyze(JSContext *cx, JSScript 
             isInlineable = false;
             break;
 
           case JSOP_THIS:
           case JSOP_GETTHISPROP:
             usesThis = true;
             break;
 
+          case JSOP_CALL:
+          case JSOP_NEW:
+            /* Only consider potentially inlineable calls here. */
+            hasCalls = true;
+            break;
+
           case JSOP_TABLESWITCH:
           case JSOP_TABLESWITCHX: {
             isInlineable = false;
             jsbytecode *pc2 = pc;
             unsigned jmplen = (op == JSOP_TABLESWITCH) ? JUMP_OFFSET_LEN : JUMPX_OFFSET_LEN;
             unsigned defaultOffset = offset + GetJumpOffset(pc, pc2);
             pc2 += jmplen;
             jsint low = GET_JUMP_OFFSET(pc2);
--- a/js/src/jsanalyze.h
+++ b/js/src/jsanalyze.h
@@ -135,16 +135,17 @@ class Script
     static const uint32 LOCAL_USE_BEFORE_DEF = uint32(-1);
     static const uint32 LOCAL_CONDITIONALLY_DEFINED = uint32(-2);
 
     bool outOfMemory;
     bool hadFailure;
     bool usesRval;
     bool usesScope;
     bool usesThis;
+    bool hasCalls;
 
     bool isInlineable;
 
     JSPackedBool *closedVars;
     JSPackedBool *closedArgs;
 
   public:
     /* Pool for allocating analysis structures which will not outlive this script. */
@@ -160,18 +161,18 @@ class Script
     bool inlineable(uint32 argc) { return isInlineable && argc == script->fun->nargs; }
 
     /* Whether there are POPV/SETRVAL bytecodes which can write to the frame's rval. */
     bool usesReturnValue() const { return usesRval; }
 
     /* Whether there are NAME bytecodes which can access the frame's scope chain. */
     bool usesScopeChain() const { return usesScope; }
 
-    /* Whether there are THIS bytecodes. */
     bool usesThisValue() const { return usesThis; }
+    bool hasFunctionCalls() const { return hasCalls; }
 
     bool hasAnalyzed() const { return !!codeArray; }
     JSScript *getScript() const { return script; }
 
     /* Accessors for bytecode information. */
 
     Bytecode& getCode(uint32 offset) {
         JS_ASSERT(offset < script->length);
--- a/js/src/jsdbgapi.cpp
+++ b/js/src/jsdbgapi.cpp
@@ -1328,17 +1328,17 @@ JS_GetScriptPrincipals(JSContext *cx, JS
 
 /*
  *  Stack Frame Iterator
  */
 JS_PUBLIC_API(JSStackFrame *)
 JS_FrameIterator(JSContext *cx, JSStackFrame **iteratorp)
 {
     if (*iteratorp == NULL)
-        *iteratorp = js_GetTopStackFrame(cx, FRAME_EXPAND_ALL);
+        *iteratorp = js_GetTopStackFrame(cx, FRAME_EXPAND_NONE);
     else
         *iteratorp = (*iteratorp)->prev();
     return *iteratorp;
 }
 
 JS_PUBLIC_API(JSScript *)
 JS_GetFrameScript(JSContext *cx, JSStackFrame *fp)
 {
--- a/js/src/jsfun.cpp
+++ b/js/src/jsfun.cpp
@@ -2881,19 +2881,25 @@ JSObject * JS_FASTCALL
 js_CloneFunctionObject(JSContext *cx, JSFunction *fun, JSObject *parent,
                        JSObject *proto)
 {
     JS_ASSERT(parent);
     JS_ASSERT(proto);
 
     JSObject *clone;
     if (cx->compartment == fun->compartment()) {
-        /* Don't clone functions with singleton types. JIT optimizations depend on this. */
+        /*
+         * Don't clone functions with singleton types, we need to ensure that
+         * there is only one object with this type. We may need to reparent the
+         * function, however.
+         */
         if (fun->getType()->singleton) {
             JS_ASSERT(fun->getType()->singleton == fun);
+            JS_ASSERT(fun->getProto() == proto);
+            fun->setParent(parent);
             return fun;
         }
 
         /*
          * The cloned function object does not need the extra JSFunction members
          * beyond JSObject as it points to fun via the private slot.
          */
         clone = NewNativeClassInstance(cx, &js_FunctionClass, proto, parent);
--- a/js/src/jsscript.h
+++ b/js/src/jsscript.h
@@ -614,16 +614,17 @@ struct JSScript {
     inline void *nativeCodeForPC(bool constructing, jsbytecode *pc);
 
     js::mjit::JITScript *getJIT(bool constructing) {
         return constructing ? jitCtor : jitNormal;
     }
 
     size_t callCount() const  { return callCount_; }
     size_t incCallCount() { return ++callCount_; }
+    size_t *addressOfCallCount() { return &callCount_; }
 
     JITScriptStatus getJITStatus(bool constructing) {
         void *addr = constructing ? jitArityCheckCtor : jitArityCheckNormal;
         if (addr == NULL)
             return JITScript_None;
         if (addr == JS_UNJITTABLE_SCRIPT)
             return JITScript_Invalid;
         return JITScript_Valid;
--- a/js/src/methodjit/Compiler.cpp
+++ b/js/src/methodjit/Compiler.cpp
@@ -78,16 +78,22 @@ using namespace js::mjit::ic;
 #if defined(JS_METHODJIT_SPEW)
 static const char *OpcodeNames[] = {
 # define OPDEF(op,val,name,token,length,nuses,ndefs,prec,format) #name,
 # include "jsopcode.tbl"
 # undef OPDEF
 };
 #endif
 
+/*
+ * Number of times a script must be called or had a backedge before we try to
+ * inline its calls.
+ */
+static const size_t CALLS_BACKEDGES_BEFORE_INLINING = 10000;
+
 mjit::Compiler::Compiler(JSContext *cx, JSStackFrame *fp,
                          const Vector<PatchableFrame> *patchFrames, bool recompiling)
   : BaseCompiler(cx),
     fp(fp), outerScript(fp->script()),
     patchFrames(patchFrames),
     scopeChain(&fp->scopeChain()),
     globalObj(scopeChain->getGlobal()),
     isConstructing(fp->isConstructing()),
@@ -117,22 +123,32 @@ mjit::Compiler::Compiler(JSContext *cx, 
     stubcc(cx, *thisFromCtor(), frame),
     debugMode_(cx->compartment->debugMode),
 #if defined JS_TRACER
     addTraceHints(cx->traceJitEnabled),
 #else
     addTraceHints(false),
 #endif
     recompiling(recompiling),
+    inlining(false),
     oomInVector(false),
     applyTricks(NoApplyTricks)
 {
     /* :FIXME: bug 637856 disabling traceJit if inference is enabled */
     if (cx->typeInferenceEnabled())
         addTraceHints = false;
+
+    /*
+     * Note: we use callCount_ to count both calls and backedges in scripts
+     * after they have been compiled and we are checking to recompile a version
+     * with inline calls. :FIXME: should remove compartment->incBackEdgeCount
+     * and do the same when deciding to initially compile.
+     */
+    if (outerScript->callCount() >= CALLS_BACKEDGES_BEFORE_INLINING)
+        inlining = true;
 }
 
 CompileStatus
 mjit::Compiler::compile()
 {
     JS_ASSERT_IF(isConstructing, !outerScript->jitCtor);
     JS_ASSERT_IF(!isConstructing, !outerScript->jitNormal);
 
@@ -265,16 +281,21 @@ mjit::Compiler::popActiveFrame()
 CompileStatus
 mjit::Compiler::performCompilation(JITScript **jitp)
 {
     outerScript = fp->script();
 
     JaegerSpew(JSpew_Scripts, "compiling script (file \"%s\") (line \"%d\") (length \"%d\")\n",
                outerScript->filename, outerScript->lineno, outerScript->length);
 
+    if (inlining) {
+        JaegerSpew(JSpew_Inlining, "inlining calls in script (file \"%s\") (line \"%d\")\n",
+                   outerScript->filename, outerScript->lineno);
+    }
+
 #ifdef JS_METHODJIT_SPEW
     Profiler prof;
     prof.start();
 #endif
 
 #ifdef JS_METHODJIT
     outerScript->debugMode = debugMode();
 #endif
@@ -582,16 +603,18 @@ mjit::Compiler::generatePrologue()
                 if (!a->analysis.argEscapes(i))
                     frame.ensureDouble(frame.getArg(i));
             } else {
                 frame.learnType(frame.getArg(i), type, false);
             }
         }
     }
 
+    recompileCheckHelper();
+
     return Compile_Okay;
 }
 
 CompileStatus
 mjit::Compiler::generateEpilogue()
 {
     return Compile_Okay;
 }
@@ -1832,17 +1855,17 @@ mjit::Compiler::generateMethod()
             bool done = false;
             if (op == JSOP_CALL) {
                 CompileStatus status = inlineNativeFunction(GET_ARGC(PC), false);
                 if (status == Compile_Okay)
                     done = true;
                 else if (status != Compile_InlineAbort)
                     return status;
             }
-            if (!done) {
+            if (!done && inlining) {
                 CompileStatus status = inlineScriptedFunction(GET_ARGC(PC), false);
                 if (status == Compile_Okay)
                     done = true;
                 else if (status != Compile_InlineAbort)
                     return status;
             }
             if (!done) {
                 JaegerSpew(JSpew_Insns, " --- SCRIPTED CALL --- \n");
@@ -2466,18 +2489,20 @@ mjit::Compiler::generateMethod()
             frame.takeReg(Registers::ReturnReg);
             frame.pushTypedPayload(JSVAL_TYPE_OBJECT, Registers::ReturnReg);
           }
           END_CASE(JSOP_LAMBDA_FC)
 
           BEGIN_CASE(JSOP_TRACE)
           BEGIN_CASE(JSOP_NOTRACE)
           {
-            if (a->analysis.jumpTarget(PC))
+            if (a->analysis.jumpTarget(PC)) {
                 interruptCheckHelper();
+                recompileCheckHelper();
+            }
           }
           END_CASE(JSOP_TRACE)
 
           BEGIN_CASE(JSOP_DEBUGGER)
             prepareStubCall(Uses(0));
             masm.move(ImmPtr(PC), Registers::ArgReg1);
             INLINE_STUBCALL(stubs::Debugger);
           END_CASE(JSOP_DEBUGGER)
@@ -2904,16 +2929,40 @@ mjit::Compiler::interruptCheckHelper()
 
     frame.sync(stubcc.masm, Uses(0));
     stubcc.masm.move(ImmPtr(PC), Registers::ArgReg1);
     OOL_STUBCALL(stubs::Interrupt);
     stubcc.rejoin(Changes(0));
 }
 
 void
+mjit::Compiler::recompileCheckHelper()
+{
+    if (!a->analysis.hasFunctionCalls() || !cx->typeInferenceEnabled())
+        return;
+
+    if (inlining) {
+        OOL_STUBCALL(stubs::RecompileForInline);
+        stubcc.rejoin(Changes(0));
+        return;
+    }
+
+    JS_ASSERT(script->callCount() < CALLS_BACKEDGES_BEFORE_INLINING);
+
+    size_t *addr = script->addressOfCallCount();
+    masm.add32(Imm32(1), AbsoluteAddress(addr));
+    Jump jump = masm.branch32(Assembler::GreaterThanOrEqual, AbsoluteAddress(addr),
+                              Imm32(CALLS_BACKEDGES_BEFORE_INLINING));
+    stubcc.linkExit(jump, Uses(0));
+
+    OOL_STUBCALL(stubs::RecompileForInline);
+    stubcc.rejoin(Changes(0));
+}
+
+void
 mjit::Compiler::addReturnSite(Label joinPoint, bool ool)
 {
     InternalCallSite site(masm.distanceOf(joinPoint), a->inlineIndex, PC,
                           CallSite::NCODE_RETURN_ID, false, ool);
     addCallSite(site);
 }
 
 void
@@ -3428,16 +3477,18 @@ mjit::Compiler::inlineCallHelper(uint32 
 
     return true;
 #endif
 }
 
 CompileStatus
 mjit::Compiler::inlineScriptedFunction(uint32 argc, bool callingNew)
 {
+    JS_ASSERT(inlining);
+
     if (!cx->typeInferenceEnabled())
         return Compile_InlineAbort;
 
     /* :XXX: Not doing inlining yet when calling 'new' or calling from 'new'. */
     if (isConstructing || callingNew)
         return Compile_InlineAbort;
 
     if (applyTricks == LazyArgsObj)
@@ -3515,16 +3566,19 @@ mjit::Compiler::inlineScriptedFunction(u
         return Compile_InlineAbort;
     }
 
     if (a->analysis.usesThisValue() && origThis->isNotType(JSVAL_TYPE_OBJECT)) {
         popActiveFrame();
         return Compile_InlineAbort;
     }
 
+    JaegerSpew(JSpew_Inlining, "inlining call to script (file \"%s\") (line \"%d\")\n",
+               script->filename, script->lineno);
+
     a->needReturnValue = needReturnValue;
     a->syncReturnValue = syncReturnValue;
     a->returnValueDouble = returnType == JSVAL_TYPE_DOUBLE;
 
     status = generateMethod();
     if (status != Compile_Okay) {
         popActiveFrame();
         if (status == Compile_Abort) {
@@ -3561,16 +3615,19 @@ mjit::Compiler::inlineScriptedFunction(u
 
     /* If we end up expanding the inline frame, it will need a return site to rejoin at. */
     addReturnSite(stubcc.masm.label(), true);
     stubcc.masm.loadPtr(Address(JSFrameReg, JSStackFrame::offsetOfPrev()), JSFrameReg);
     stubcc.masm.storeValueFromComponents(JSReturnReg_Type, JSReturnReg_Data,
                                          frame.addressOf(frame.peek(-1)));
     stubcc.rejoin(Changes(1));
 
+    JaegerSpew(JSpew_Inlining, "finished inlining call to script (file \"%s\") (line \"%d\")\n",
+               script->filename, script->lineno);
+
     return Compile_Okay;
 }
 
 /*
  * This function must be called immediately after any instruction which could
  * cause a new JSStackFrame to be pushed and could lead to a new debug trap
  * being set. This includes any API callbacks and any scripted or native call.
  */
--- a/js/src/methodjit/Compiler.h
+++ b/js/src/methodjit/Compiler.h
@@ -434,16 +434,17 @@ class Compiler : public BaseCompiler
 #ifdef JS_MONOIC
     Label argsCheckStub;
     Label argsCheckFallthrough;
     Jump argsCheckJump;
 #endif
     bool debugMode_;
     bool addTraceHints;
     bool recompiling;
+    bool inlining;
     bool oomInVector;       // True if we have OOM'd appending to a vector. 
     enum { NoApplyTricks, LazyArgsObj } applyTricks;
 
     Compiler *thisFromCtor() { return this; }
 
     friend class CompilerAllocPolicy;
   public:
     // Special atom index used to indicate that the atom is 'length'. This
@@ -547,16 +548,17 @@ class Compiler : public BaseCompiler
     void jsop_this();
     void emitReturn(FrameEntry *fe);
     void emitFinalReturn(Assembler &masm);
     void loadReturnValue(Assembler *masm, FrameEntry *fe);
     void emitReturnValue(Assembler *masm, FrameEntry *fe);
     void emitInlineReturnValue(FrameEntry *fe);
     void dispatchCall(VoidPtrStubUInt32 stub, uint32 argc);
     void interruptCheckHelper();
+    void recompileCheckHelper();
     void emitUncachedCall(uint32 argc, bool callingNew);
     void checkCallApplySpeculation(uint32 callImmArgc, uint32 speculatedArgc,
                                    FrameEntry *origCallee, FrameEntry *origThis,
                                    MaybeRegisterID origCalleeType, RegisterID origCalleeData,
                                    MaybeRegisterID origThisType, RegisterID origThisData,
                                    Jump *uncachedCallSlowRejoin, CallPatchInfo *uncachedCallPatch);
     bool inlineCallHelper(uint32 argc, bool callingNew);
     void fixPrimitiveReturn(Assembler *masm, FrameEntry *fe);
--- a/js/src/methodjit/Logging.cpp
+++ b/js/src/methodjit/Logging.cpp
@@ -81,16 +81,17 @@ js::JMCheckLogging()
 #ifdef DEBUG
             "  jsops         JS opcodes\n"
 #endif
             "  insns         JS opcodes and generated insns\n"
             "  vmframe       VMFrame contents\n"
             "  pics          PIC patching activity\n"
             "  slowcalls     Calls to slow path functions\n"
             "  regalloc      Register allocation behavior\n"
+            "  inlin         Call inlining behavior\n"
             "  recompile     Dynamic recompilations\n"
             "  full          everything\n"
             "  notrace       disable trace hints\n"
             "\n"
         );
         exit(0);
         /*NOTREACHED*/
     }
@@ -111,16 +112,18 @@ js::JMCheckLogging()
     if (strstr(env, "pics"))
         LoggingBits |= (1 << uint32(JSpew_PICs));
     if (strstr(env, "slowcalls"))
         LoggingBits |= (1 << uint32(JSpew_SlowCalls));
     if (strstr(env, "regalloc"))
         LoggingBits |= (1 << uint32(JSpew_Regalloc));
     if (strstr(env, "recompile"))
         LoggingBits |= (1 << uint32(JSpew_Recompile));
+    if (strstr(env, "inlin"))
+        LoggingBits |= (1 << uint32(JSpew_Inlining));
     if (strstr(env, "full"))
         LoggingBits |= 0xFFFFFFFF;
 }
 
 bool
 js::IsJaegerSpewChannelActive(JaegerSpewChannel channel)
 {
     JS_ASSERT(LoggingChecked);
--- a/js/src/methodjit/Logging.h
+++ b/js/src/methodjit/Logging.h
@@ -53,16 +53,17 @@ namespace js {
     _(Scripts)              \
     _(Prof)                 \
     _(JSOps)                \
     _(Insns)                \
     _(VMFrame)              \
     _(PICs)                 \
     _(SlowCalls)            \
     _(Regalloc)             \
+    _(Inlining)             \
     _(Recompile)
 
 enum JaegerSpewChannel {
 #define _(name) JSpew_##name,
     JSPEW_CHAN_MAP(_)
 #undef  _
     JSpew_Terminator
 };
--- a/js/src/methodjit/StubCalls.cpp
+++ b/js/src/methodjit/StubCalls.cpp
@@ -47,16 +47,17 @@
 #include "jsxml.h"
 #include "jsstaticcheck.h"
 #include "jsbool.h"
 #include "assembler/assembler/MacroAssemblerCodeRef.h"
 #include "jsiter.h"
 #include "jstypes.h"
 #include "methodjit/Compiler.h"
 #include "methodjit/StubCalls.h"
+#include "methodjit/Retcon.h"
 
 #include "jsinterpinlines.h"
 #include "jspropertycache.h"
 #include "jspropertycacheinlines.h"
 #include "jsscopeinlines.h"
 #include "jsscriptinlines.h"
 #include "jsstrinlines.h"
 #include "jsobjinlines.h"
@@ -1296,16 +1297,24 @@ stubs::Debugger(VMFrame &f, jsbytecode *
 void JS_FASTCALL
 stubs::Interrupt(VMFrame &f, jsbytecode *pc)
 {
     if (!js_HandleExecutionInterrupt(f.cx))
         THROW();
 }
 
 void JS_FASTCALL
+stubs::RecompileForInline(VMFrame &f)
+{
+    Recompiler recompiler(f.cx, f.script());
+    if (!recompiler.recompile())
+        THROW();
+}
+
+void JS_FASTCALL
 stubs::Trap(VMFrame &f, uint32 trapTypes)
 {
     Value rval;
 
     /*
      * Trap may be called for a single-step interrupt trap and/or a
      * regular trap. Try the single-step first, and if it lets control
      * flow through or does not exist, do the regular trap.
--- a/js/src/methodjit/StubCalls.h
+++ b/js/src/methodjit/StubCalls.h
@@ -54,16 +54,17 @@ typedef enum JSTrapType {
 } JSTrapType;
 
 void JS_FASTCALL This(VMFrame &f);
 JSObject * JS_FASTCALL NewInitArray(VMFrame &f, uint32 count);
 JSObject * JS_FASTCALL NewInitObject(VMFrame &f, JSObject *base);
 void JS_FASTCALL Trap(VMFrame &f, uint32 trapTypes);
 void JS_FASTCALL Debugger(VMFrame &f, jsbytecode *pc);
 void JS_FASTCALL Interrupt(VMFrame &f, jsbytecode *pc);
+void JS_FASTCALL RecompileForInline(VMFrame &f);
 void JS_FASTCALL InitElem(VMFrame &f, uint32 last);
 void JS_FASTCALL InitProp(VMFrame &f, JSAtom *atom);
 void JS_FASTCALL InitMethod(VMFrame &f, JSAtom *atom);
 
 void JS_FASTCALL HitStackQuota(VMFrame &f);
 void * JS_FASTCALL FixupArity(VMFrame &f, uint32 argc);
 void * JS_FASTCALL CompileFunction(VMFrame &f, uint32 argc);
 void JS_FASTCALL SlowNew(VMFrame &f, uint32 argc);