Bug 826148 - Part 3: Jaeger IC (r=bhackett)
authorShu-yu Guo <shu@rfrn.org>
Thu, 10 Jan 2013 13:04:04 -0800
changeset 118629 a5e3c3dd6ab2ee7a210438db06fd7b13944a2bcc
parent 118628 c8b493e0551523f3ccea38bad7264910f24af7bf
child 118630 8120c8952c4059b7ee4f610c11bbad418d2b5830
push idunknown
push userunknown
push dateunknown
reviewersbhackett
bugs826148
milestone21.0a1
Bug 826148 - Part 3: Jaeger IC (r=bhackett)
js/src/methodjit/Compiler.cpp
js/src/methodjit/Compiler.h
js/src/methodjit/InvokeHelpers.cpp
js/src/methodjit/MonoIC.cpp
js/src/methodjit/MonoIC.h
js/src/methodjit/StubCalls-inl.h
js/src/methodjit/StubCalls.h
--- a/js/src/methodjit/Compiler.cpp
+++ b/js/src/methodjit/Compiler.cpp
@@ -1632,16 +1632,17 @@ mjit::Compiler::finishThisUp()
 
         stubCode.patch(from.addrLabel, &to);
     }
 
     ic::CallICInfo *jitCallICs = (ic::CallICInfo *)cursor;
     chunk->nCallICs = callICs.length();
     cursor += sizeof(ic::CallICInfo) * chunk->nCallICs;
     for (size_t i = 0; i < chunk->nCallICs; i++) {
+        jitCallICs[i].funGuardLabel = fullCode.locationOf(callICs[i].funGuardLabel);
         jitCallICs[i].funGuard = fullCode.locationOf(callICs[i].funGuard);
         jitCallICs[i].funJump = fullCode.locationOf(callICs[i].funJump);
         jitCallICs[i].slowPathStart = stubCode.locationOf(callICs[i].slowPathStart);
         jitCallICs[i].typeMonitored = callICs[i].typeMonitored;
 
         /* Compute the hot call offset. */
         uint32_t offset = fullCode.locationOf(callICs[i].hotJump) -
                         fullCode.locationOf(callICs[i].funGuard);
@@ -4425,16 +4426,17 @@ mjit::Compiler::inlineCallHelper(uint32_
     /* Reserve space just before initialization of funGuard. */
     RESERVE_IC_SPACE(masm);
 
     /*
      * Guard on the callee identity. This misses on the first run. If the
      * callee is scripted, compiled/compilable, and argc == nargs, then this
      * guard is patched, and the compiled code address is baked in.
      */
+    callIC.funGuardLabel = masm.label();
     Jump j = masm.branchPtrWithPatch(Assembler::NotEqual, icCalleeData, callIC.funGuard);
     callIC.funJump = j;
 
     /* Reserve space just before initialization of slowPathStart. */
     RESERVE_OOL_SPACE(stubcc.masm);
 
     Jump rejoin1, rejoin2;
     {
@@ -4624,16 +4626,21 @@ mjit::Compiler::inlineScriptedFunction(u
     JS_ASSERT(inlining());
 
     /* We already know which frames we are inlining at each PC, so scan the list of inline frames. */
     bool calleeMultipleReturns = false;
     Vector<JSScript *> inlineCallees(CompilerAllocPolicy(cx, *this));
     for (unsigned i = 0; i < ssa.numFrames(); i++) {
         if (ssa.iterFrame(i).parent == a->inlineIndex && ssa.iterFrame(i).parentpc == PC) {
             JSScript *script_ = ssa.iterFrame(i).script;
+
+            /* Don't inline if any of the callees should be cloned at callsite. */
+            if (script_->function()->isCloneAtCallsite())
+                return Compile_InlineAbort;
+
             inlineCallees.append(script_);
             if (script_->analysis()->numReturnSites() > 1)
                 calleeMultipleReturns = true;
         }
     }
 
     if (inlineCallees.empty())
         return Compile_InlineAbort;
--- a/js/src/methodjit/Compiler.h
+++ b/js/src/methodjit/Compiler.h
@@ -115,16 +115,17 @@ class Compiler : public BaseCompiler
     /* InlineFrameAssembler wants to see this. */
   public:
     struct CallGenInfo {
         /*
          * These members map to members in CallICInfo. See that structure for
          * more comments.
          */
         uint32_t     callIndex;
+        Label        funGuardLabel;
         DataLabelPtr funGuard;
         Jump         funJump;
         Jump         hotJump;
         Call         oolCall;
         Label        joinPoint;
         Label        slowJoinPoint;
         Label        slowPathStart;
         Label        hotPathLabel;
--- a/js/src/methodjit/InvokeHelpers.cpp
+++ b/js/src/methodjit/InvokeHelpers.cpp
@@ -125,38 +125,60 @@ FindExceptionHandler(JSContext *cx)
 
     return NULL;
 }
 
 /*
  * Clean up a frame and return.
  */
 
+static inline bool
+MaybeCloneAndPatchCallee(JSContext *cx, CallArgs args, HandleScript script, jsbytecode *pc)
+{
+    if (cx->typeInferenceEnabled() && !args.calleev().isPrimitive() &&
+        args.callee().isFunction() && args.callee().toFunction()->isCloneAtCallsite())
+    {
+        RootedFunction fun(cx, args.callee().toFunction());
+        fun = CloneFunctionAtCallsite(cx, fun, script, pc);
+        if (!fun)
+            return false;
+        args.setCallee(ObjectValue(*fun));
+    }
+
+    return true;
+}
+
 void JS_FASTCALL
 stubs::SlowCall(VMFrame &f, uint32_t argc)
 {
     if (*f.regs.pc == JSOP_FUNAPPLY && !GuardFunApplyArgumentsOptimization(f.cx))
         THROW();
 
     CallArgs args = CallArgsFromSp(argc, f.regs.sp);
+    RootedScript fscript(f.cx, f.script());
+
+    if (!MaybeCloneAndPatchCallee(f.cx, args, fscript, f.pc()))
+        THROW();
     if (!InvokeKernel(f.cx, args))
         THROW();
 
-    RootedScript fscript(f.cx, f.script());
     types::TypeScript::Monitor(f.cx, fscript, f.pc(), args.rval());
 }
 
 void JS_FASTCALL
 stubs::SlowNew(VMFrame &f, uint32_t argc)
 {
     CallArgs args = CallArgsFromSp(argc, f.regs.sp);
+    RootedScript fscript(f.cx, f.script());
+
+    if (!MaybeCloneAndPatchCallee(f.cx, args, fscript, f.pc()))
+        THROW();
     if (!InvokeConstructorKernel(f.cx, args))
         THROW();
 
-    RootedScript fscript(f.cx, f.script());
     types::TypeScript::Monitor(f.cx, fscript, f.pc(), args.rval());
 }
 
 static inline bool
 CheckStackQuota(VMFrame &f)
 {
     JS_ASSERT(f.regs.stackDepth() == 0);
 
@@ -205,16 +227,20 @@ stubs::FixupArity(VMFrame &f, uint32_t n
     RootedScript script(cx, fun->nonLazyScript());
     void *ncode = oldfp->nativeReturnAddress();
 
     /* Pop the inline frame. */
     f.regs.popPartialFrame((Value *)oldfp);
 
     /* Reserve enough space for a callee frame. */
     CallArgs args = CallArgsFromSp(nactual, f.regs.sp);
+    if (fun->isCallsiteClone()) {
+        JS_ASSERT(args.callee().toFunction() == fun->getExtendedSlot(0).toObject().toFunction());
+        args.setCallee(ObjectValue(*fun));
+    }
     StackFrame *fp = cx->stack.getFixupFrame(cx, DONT_REPORT_ERROR, args, fun,
                                              script, ncode, initial, &f.stackLimit);
 
     if (!fp) {
         f.regs.updateForNcode(f.jit(), ncode);
         js_ReportOverRecursed(cx);
         THROWV(NULL);
     }
@@ -282,17 +308,17 @@ static inline bool
 UncachedInlineCall(VMFrame &f, InitialFrameFlags initial,
                    void **pret, bool *unjittable, uint32_t argc)
 {
     AssertCanGC();
     JSContext *cx = f.cx;
     CallArgs args = CallArgsFromSp(argc, f.regs.sp);
     RootedFunction newfun(cx, args.callee().toFunction());
 
-    RootedScript newscript(cx, JSFunction::getOrCreateScript(cx, newfun));
+    RootedScript newscript(cx, newfun->nonLazyScript());
     if (!newscript)
         return false;
 
     bool construct = InitialFrameFlagsAreConstructing(initial);
 
     RootedScript fscript(cx, f.script());
     bool newType = construct && cx->typeInferenceEnabled() &&
         types::UseNewType(cx, fscript, f.pc());
@@ -390,25 +416,28 @@ stubs::UncachedNew(VMFrame &f, uint32_t 
 }
 
 void
 stubs::UncachedNewHelper(VMFrame &f, uint32_t argc, UncachedCallResult &ucr)
 {
     ucr.init();
     JSContext *cx = f.cx;
     CallArgs args = CallArgsFromSp(argc, f.regs.sp);
+    RootedScript fscript(cx, f.script());
+
+    if (!ucr.setFunction(cx, args, fscript, f.pc()))
+        THROW();
 
     /* Try to do a fast inline call before the general Invoke path. */
-    if (IsFunctionObject(args.calleev(), ucr.fun.address()) && ucr.fun->isInterpretedConstructor()) {
+    if (ucr.fun && ucr.fun->isInterpretedConstructor()) {
         if (!UncachedInlineCall(f, INITIAL_CONSTRUCT, &ucr.codeAddr, &ucr.unjittable, argc))
             THROW();
     } else {
         if (!InvokeConstructorKernel(cx, args))
             THROW();
-        RootedScript fscript(cx, f.script());
         types::TypeScript::Monitor(f.cx, fscript, f.pc(), args.rval());
     }
 }
 
 void * JS_FASTCALL
 stubs::UncachedCall(VMFrame &f, uint32_t argc)
 {
     UncachedCallResult ucr(f.cx);
@@ -448,18 +477,22 @@ stubs::Eval(VMFrame &f, uint32_t argc)
 
 void
 stubs::UncachedCallHelper(VMFrame &f, uint32_t argc, bool lowered, UncachedCallResult &ucr)
 {
     ucr.init();
 
     JSContext *cx = f.cx;
     CallArgs args = CallArgsFromSp(argc, f.regs.sp);
+    RootedScript fscript(cx, f.script());
 
-    if (IsFunctionObject(args.calleev(), ucr.fun.address())) {
+    if (!ucr.setFunction(cx, args, fscript, f.pc()))
+        THROW();
+
+    if (ucr.fun) {
         if (ucr.fun->isInterpreted()) {
             InitialFrameFlags initial = lowered ? INITIAL_LOWERED : INITIAL_NONE;
             if (!UncachedInlineCall(f, initial, &ucr.codeAddr, &ucr.unjittable, argc))
                 THROW();
             return;
         }
 
         if (ucr.fun->isNative()) {
@@ -469,17 +502,16 @@ stubs::UncachedCallHelper(VMFrame &f, ui
             types::TypeScript::Monitor(f.cx, fscript, f.pc(), args.rval());
             return;
         }
     }
 
     if (!InvokeKernel(f.cx, args))
         THROW();
 
-    RootedScript fscript(cx, f.script());
     types::TypeScript::Monitor(f.cx, fscript, f.pc(), args.rval());
     return;
 }
 
 static void
 RemoveOrphanedNative(JSContext *cx, StackFrame *fp)
 {
     /*
--- a/js/src/methodjit/MonoIC.cpp
+++ b/js/src/methodjit/MonoIC.cpp
@@ -1024,23 +1024,23 @@ class CallCompiler : public BaseCompiler
         if (!linker.verifyRange(f.chunk())) {
             disable();
             return true;
         }
 
         linker.link(claspGuard, ic.slowPathStart);
         linker.link(funGuard, ic.slowPathStart);
         linker.link(done, ic.funGuard.labelAtOffset(ic.hotPathOffset));
-        JSC::CodeLocationLabel cs = linker.finalize(f);
+        ic.funJumpTarget = linker.finalize(f);
 
         JaegerSpew(JSpew_PICs, "generated CALL closure stub %p (%lu bytes)\n",
-                   cs.executableAddress(), (unsigned long) masm.size());
+                   ic.funJumpTarget.executableAddress(), (unsigned long) masm.size());
 
         Repatcher repatch(f.chunk());
-        repatch.relink(ic.funJump, cs);
+        repatch.relink(ic.funJump, ic.funJumpTarget);
 
         return true;
     }
 
     bool generateNativeStub()
     {
         /* Snapshot the frameDepth before SplatApplyArgs modifies it. */
         unsigned initialFrameDepth = f.regs.sp - f.fp()->slots();
@@ -1204,20 +1204,77 @@ class CallCompiler : public BaseCompiler
             return true;
         }
 
         linker.patchJump(ic.nativeRejoin());
 
         ic.fastGuardedNative = fun;
 
         linker.link(funGuard, ic.slowPathStart);
+        ic.funJumpTarget = linker.finalize(f);
+
+        JaegerSpew(JSpew_PICs, "generated native CALL stub %p (%lu bytes)\n",
+                   ic.funJumpTarget.executableAddress(), (unsigned long) masm.size());
+
+        Repatcher repatch(f.chunk());
+        repatch.relink(ic.funJump, ic.funJumpTarget);
+
+        return true;
+    }
+
+    bool generateCallsiteCloneStub(HandleFunction original, HandleFunction fun)
+    {
+        AutoAssertNoGC nogc;
+
+        Assembler masm;
+
+        // If we have a callsite clone, we do the folowing hack:
+        //
+        //  1) Patch funJump to a stub which guards on the identity of the
+        //     original function. If this guard fails, we jump to the original
+        //     funJump target.
+        //  2) Load the clone into the callee register.
+        //  3) Jump *back* to funGuard.
+        //
+        // For the inline path, hopefully we will succeed upon jumping back to
+        // funGuard after loading the clone.
+        //
+        // This hack is not ideal, as we can actually fail the first funGuard
+        // twice: we fail the first funGuard, pass the second funGuard, load
+        // the clone, and jump back to the first funGuard. We fail the first
+        // funGuard again, and this time we also fail the second funGuard,
+        // since a function's clone is never equal to itself. Finally, we jump
+        // to the original funJump target.
+
+        // Guard on the original function's identity.
+        Jump originalGuard = masm.branchPtr(Assembler::NotEqual, ic.funObjReg, ImmPtr(original));
+
+        // Load the clone.
+        masm.move(ImmPtr(fun), ic.funObjReg);
+
+        // Jump back to the first fun guard.
+        Jump done = masm.jump();
+
+        LinkerHelper linker(masm, JSC::JAEGER_CODE);
+        JSC::ExecutablePool *ep = linker.init(f.cx);
+        if (!ep)
+            return false;
+
+        if (!linker.verifyRange(f.chunk())) {
+            disable();
+            return true;
+        }
+
+        linker.link(originalGuard, !!ic.funJumpTarget ? ic.funJumpTarget : ic.slowPathStart);
+        linker.link(done, ic.funGuardLabel);
         JSC::CodeLocationLabel start = linker.finalize(f);
 
-        JaegerSpew(JSpew_PICs, "generated native CALL stub %p (%lu bytes)\n",
+        JaegerSpew(JSpew_PICs, "generated CALL clone stub %p (%lu bytes)\n",
                    start.executableAddress(), (unsigned long) masm.size());
+        JaegerSpew(JSpew_PICs, "guarding %p with clone %p\n", original.get(), fun.get());
 
         Repatcher repatch(f.chunk());
         repatch.relink(ic.funJump, start);
 
         return true;
     }
 
     void *update()
@@ -1298,16 +1355,19 @@ class CallCompiler : public BaseCompiler
                 if (!generateStubForClosures(fun))
                     THROWV(NULL);
             } else {
                 if (!generateFullCallStub(script, flags))
                     THROWV(NULL);
             }
         }
 
+        if (ucr.original && !generateCallsiteCloneStub(ucr.original, ucr.fun))
+            THROWV(NULL);
+
         return ucr.codeAddr;
     }
 };
 
 } // namespace mjit
 } // namespace js
 
 void * JS_FASTCALL
--- a/js/src/methodjit/MonoIC.h
+++ b/js/src/methodjit/MonoIC.h
@@ -160,26 +160,35 @@ struct CallICInfo {
     JSObject *fastGuardedObject;
     JSObject *fastGuardedNative;
 
     /* Return site for scripted calls at this site, with PC and inlining state. */
     CallSite *call;
 
     FrameSize frameSize;
 
+    /* Label to the function object identity guard. */
+    JSC::CodeLocationLabel funGuardLabel;
+
     /* Function object identity guard. */
     JSC::CodeLocationDataLabelPtr funGuard;
 
     /* Starting point for all slow call paths. */
     JSC::CodeLocationLabel slowPathStart;
 
     /* Inline to OOL jump, redirected by stubs. */
     JSC::CodeLocationJump funJump;
 
     /*
+     * Target of the above jump, remembered so that if we need to generate a
+     * callsite clone stub we can redirect to the original funJump target.
+     */
+    JSC::CodeLocationLabel funJumpTarget;
+
+    /*
      * If an Ion stub has been generated, its guard may be linked to another
      * stub. The guard location is stored in this label.
      */
     bool hasIonStub_;
     JSC::JITCode lastOolCode_;
     JSC::CodeLocationJump lastOolJump_;
 
     /* Offset to inline scripted call, from funGuard. */
--- a/js/src/methodjit/StubCalls-inl.h
+++ b/js/src/methodjit/StubCalls-inl.h
@@ -24,13 +24,34 @@ ThrowException(VMFrame &f)
 static inline void
 ReportAtomNotDefined(JSContext *cx, JSAtom *atom)
 {
     JSAutoByteString printable;
     if (js_AtomToPrintableString(cx, atom, &printable))
         js_ReportIsNotDefined(cx, printable.ptr());
 }
 
+inline bool
+stubs::UncachedCallResult::setFunction(JSContext *cx, CallArgs &args,
+                                       HandleScript callScript, jsbytecode *callPc)
+{
+    if (!IsFunctionObject(args.calleev(), fun.address()))
+        return true;
+
+    if (fun->isInterpretedLazy() && !JSFunction::getOrCreateScript(cx, fun))
+        return false;
+
+    if (cx->typeInferenceEnabled() && fun->isCloneAtCallsite()) {
+        original = fun;
+        fun = CloneFunctionAtCallsite(cx, original, callScript, callPc);
+        if (!fun)
+            return false;
+        args.setCallee(ObjectValue(*fun));
+    }
+
+    return true;
+}
+
 } /* namespace mjit */
 } /* namespace js */
 
 #endif /* jslogic_h__ */
 
--- a/js/src/methodjit/StubCalls.h
+++ b/js/src/methodjit/StubCalls.h
@@ -3,16 +3,17 @@
  *
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #if !defined jslogic_h__ && defined JS_METHODJIT
 #define jslogic_h__
 
+#include "jsfuninlines.h"
 #include "MethodJIT.h"
 
 namespace js {
 namespace mjit {
 namespace stubs {
 
 typedef enum JSTrapType {
     JSTRAP_NONE = 0,
@@ -53,26 +54,31 @@ void JS_FASTCALL ScriptProbeOnlyEpilogue
  *   (1) The function was executed in the interpreter. Then all fields
  *       are NULL except unjittable.
  *
  *   (2) The function was not executed, and the function has been compiled
  *       to JM native code. Then all fields are non-NULL.
  */
 struct UncachedCallResult {
     RootedFunction fun;        // callee function
+    RootedFunction original;   // NULL if fun is not a callsite clone, else
+                               // points to the original function.
     void           *codeAddr;  // code address of compiled callee function
     bool           unjittable; // did we try to JIT and fail?
 
-    UncachedCallResult(JSContext *cx) : fun(cx) {}
+    UncachedCallResult(JSContext *cx) : fun(cx), original(cx) {}
 
     void init() {
         fun = NULL;
+        original = NULL;
         codeAddr = NULL;
         unjittable = false;
     }
+    inline bool setFunction(JSContext *cx, CallArgs &args,
+                            HandleScript callScript, jsbytecode *callPc);
 };
 
 /*
  * Helper functions for stubs and IC functions for calling functions.
  * These functions either execute the function, return a native code
  * pointer that can be used to call the function, or throw.
  */
 void UncachedCallHelper(VMFrame &f, uint32_t argc, bool lowered, UncachedCallResult &ucr);