Bug 826148 - Part 3: Jaeger IC (r=bhackett)
☠☠ backed out by 6cda85d6e4f6 ☠ ☠
authorShu-yu Guo <shu@rfrn.org>
Thu, 10 Jan 2013 13:04:04 -0800
changeset 118504 1db5b4e596492d56613ce019ae3f6126e5bd6816
parent 118503 57bf735f3e186a1501fe2fe01b46bc1ef41d0ca2
child 118505 e520c411aed6a09e7d2fade06aa949cd018e97f3
push id24166
push userMs2ger@gmail.com
push dateFri, 11 Jan 2013 13:57:41 +0000
treeherdermozilla-central@63c4b0f66a0c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbhackett
bugs826148
milestone21.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 826148 - Part 3: Jaeger IC (r=bhackett)
js/src/methodjit/Compiler.cpp
js/src/methodjit/Compiler.h
js/src/methodjit/InvokeHelpers.cpp
js/src/methodjit/MonoIC.cpp
js/src/methodjit/MonoIC.h
js/src/methodjit/StubCalls-inl.h
js/src/methodjit/StubCalls.h
--- a/js/src/methodjit/Compiler.cpp
+++ b/js/src/methodjit/Compiler.cpp
@@ -1632,16 +1632,17 @@ mjit::Compiler::finishThisUp()
 
         stubCode.patch(from.addrLabel, &to);
     }
 
     ic::CallICInfo *jitCallICs = (ic::CallICInfo *)cursor;
     chunk->nCallICs = callICs.length();
     cursor += sizeof(ic::CallICInfo) * chunk->nCallICs;
     for (size_t i = 0; i < chunk->nCallICs; i++) {
+        jitCallICs[i].funGuardLabel = fullCode.locationOf(callICs[i].funGuardLabel);
         jitCallICs[i].funGuard = fullCode.locationOf(callICs[i].funGuard);
         jitCallICs[i].funJump = fullCode.locationOf(callICs[i].funJump);
         jitCallICs[i].slowPathStart = stubCode.locationOf(callICs[i].slowPathStart);
         jitCallICs[i].typeMonitored = callICs[i].typeMonitored;
 
         /* Compute the hot call offset. */
         uint32_t offset = fullCode.locationOf(callICs[i].hotJump) -
                         fullCode.locationOf(callICs[i].funGuard);
@@ -4425,16 +4426,17 @@ mjit::Compiler::inlineCallHelper(uint32_
     /* Reserve space just before initialization of funGuard. */
     RESERVE_IC_SPACE(masm);
 
     /*
      * Guard on the callee identity. This misses on the first run. If the
      * callee is scripted, compiled/compilable, and argc == nargs, then this
      * guard is patched, and the compiled code address is baked in.
      */
+    callIC.funGuardLabel = masm.label();
     Jump j = masm.branchPtrWithPatch(Assembler::NotEqual, icCalleeData, callIC.funGuard);
     callIC.funJump = j;
 
     /* Reserve space just before initialization of slowPathStart. */
     RESERVE_OOL_SPACE(stubcc.masm);
 
     Jump rejoin1, rejoin2;
     {
@@ -4624,16 +4626,21 @@ mjit::Compiler::inlineScriptedFunction(u
     JS_ASSERT(inlining());
 
     /* We already know which frames we are inlining at each PC, so scan the list of inline frames. */
     bool calleeMultipleReturns = false;
     Vector<JSScript *> inlineCallees(CompilerAllocPolicy(cx, *this));
     for (unsigned i = 0; i < ssa.numFrames(); i++) {
         if (ssa.iterFrame(i).parent == a->inlineIndex && ssa.iterFrame(i).parentpc == PC) {
             JSScript *script_ = ssa.iterFrame(i).script;
+
+            /* Don't inline if any of the callees should be cloned at callsite. */
+            if (script_->function()->isCloneAtCallsite())
+                return Compile_InlineAbort;
+
             inlineCallees.append(script_);
             if (script_->analysis()->numReturnSites() > 1)
                 calleeMultipleReturns = true;
         }
     }
 
     if (inlineCallees.empty())
         return Compile_InlineAbort;
--- a/js/src/methodjit/Compiler.h
+++ b/js/src/methodjit/Compiler.h
@@ -115,16 +115,17 @@ class Compiler : public BaseCompiler
     /* InlineFrameAssembler wants to see this. */
   public:
     struct CallGenInfo {
         /*
          * These members map to members in CallICInfo. See that structure for
          * more comments.
          */
         uint32_t     callIndex;
+        Label        funGuardLabel;
         DataLabelPtr funGuard;
         Jump         funJump;
         Jump         hotJump;
         Call         oolCall;
         Label        joinPoint;
         Label        slowJoinPoint;
         Label        slowPathStart;
         Label        hotPathLabel;
--- a/js/src/methodjit/InvokeHelpers.cpp
+++ b/js/src/methodjit/InvokeHelpers.cpp
@@ -125,38 +125,60 @@ FindExceptionHandler(JSContext *cx)
 
     return NULL;
 }
 
 /*
  * Clean up a frame and return.
  */
 
+static inline bool
+MaybeCloneAndPatchCallee(JSContext *cx, CallArgs args, HandleScript script, jsbytecode *pc)
+{
+    if (cx->typeInferenceEnabled() &&
+        args.callee().isFunction() && args.callee().toFunction()->isCloneAtCallsite())
+    {
+        RootedFunction fun(cx, args.callee().toFunction());
+        fun = CloneFunctionAtCallsite(cx, fun, script, pc);
+        if (!fun)
+            return false;
+        args.setCallee(ObjectValue(*fun));
+    }
+
+    return true;
+}
+
 void JS_FASTCALL
 stubs::SlowCall(VMFrame &f, uint32_t argc)
 {
     if (*f.regs.pc == JSOP_FUNAPPLY && !GuardFunApplyArgumentsOptimization(f.cx))
         THROW();
 
     CallArgs args = CallArgsFromSp(argc, f.regs.sp);
+    RootedScript fscript(f.cx, f.script());
+
+    if (!MaybeCloneAndPatchCallee(f.cx, args, fscript, f.pc()))
+        THROW();
     if (!InvokeKernel(f.cx, args))
         THROW();
 
-    RootedScript fscript(f.cx, f.script());
     types::TypeScript::Monitor(f.cx, fscript, f.pc(), args.rval());
 }
 
 void JS_FASTCALL
 stubs::SlowNew(VMFrame &f, uint32_t argc)
 {
     CallArgs args = CallArgsFromSp(argc, f.regs.sp);
+    RootedScript fscript(f.cx, f.script());
+
+    if (!MaybeCloneAndPatchCallee(f.cx, args, fscript, f.pc()))
+        THROW();
     if (!InvokeConstructorKernel(f.cx, args))
         THROW();
 
-    RootedScript fscript(f.cx, f.script());
     types::TypeScript::Monitor(f.cx, fscript, f.pc(), args.rval());
 }
 
 static inline bool
 CheckStackQuota(VMFrame &f)
 {
     JS_ASSERT(f.regs.stackDepth() == 0);
 
@@ -205,16 +227,20 @@ stubs::FixupArity(VMFrame &f, uint32_t n
     RootedScript script(cx, fun->nonLazyScript());
     void *ncode = oldfp->nativeReturnAddress();
 
     /* Pop the inline frame. */
     f.regs.popPartialFrame((Value *)oldfp);
 
     /* Reserve enough space for a callee frame. */
     CallArgs args = CallArgsFromSp(nactual, f.regs.sp);
+    if (fun->isCallsiteClone()) {
+        JS_ASSERT(args.callee().toFunction() == fun->getExtendedSlot(0).toObject().toFunction());
+        args.setCallee(ObjectValue(*fun));
+    }
     StackFrame *fp = cx->stack.getFixupFrame(cx, DONT_REPORT_ERROR, args, fun,
                                              script, ncode, initial, &f.stackLimit);
 
     if (!fp) {
         f.regs.updateForNcode(f.jit(), ncode);
         js_ReportOverRecursed(cx);
         THROWV(NULL);
     }
@@ -282,17 +308,17 @@ static inline bool
 UncachedInlineCall(VMFrame &f, InitialFrameFlags initial,
                    void **pret, bool *unjittable, uint32_t argc)
 {
     AssertCanGC();
     JSContext *cx = f.cx;
     CallArgs args = CallArgsFromSp(argc, f.regs.sp);
     RootedFunction newfun(cx, args.callee().toFunction());
 
-    RootedScript newscript(cx, JSFunction::getOrCreateScript(cx, newfun));
+    RootedScript newscript(cx, newfun->nonLazyScript());
     if (!newscript)
         return false;
 
     bool construct = InitialFrameFlagsAreConstructing(initial);
 
     RootedScript fscript(cx, f.script());
     bool newType = construct && cx->typeInferenceEnabled() &&
         types::UseNewType(cx, fscript, f.pc());
@@ -390,25 +416,28 @@ stubs::UncachedNew(VMFrame &f, uint32_t 
 }
 
 void
 stubs::UncachedNewHelper(VMFrame &f, uint32_t argc, UncachedCallResult &ucr)
 {
     ucr.init();
     JSContext *cx = f.cx;
     CallArgs args = CallArgsFromSp(argc, f.regs.sp);
+    RootedScript fscript(cx, f.script());
+
+    if (!ucr.setFunction(cx, args, fscript, f.pc()))
+        THROW();
 
     /* Try to do a fast inline call before the general Invoke path. */
-    if (IsFunctionObject(args.calleev(), ucr.fun.address()) && ucr.fun->isInterpretedConstructor()) {
+    if (ucr.fun && ucr.fun->isInterpretedConstructor()) {
         if (!UncachedInlineCall(f, INITIAL_CONSTRUCT, &ucr.codeAddr, &ucr.unjittable, argc))
             THROW();
     } else {
         if (!InvokeConstructorKernel(cx, args))
             THROW();
-        RootedScript fscript(cx, f.script());
         types::TypeScript::Monitor(f.cx, fscript, f.pc(), args.rval());
     }
 }
 
 void * JS_FASTCALL
 stubs::UncachedCall(VMFrame &f, uint32_t argc)
 {
     UncachedCallResult ucr(f.cx);
@@ -448,18 +477,22 @@ stubs::Eval(VMFrame &f, uint32_t argc)
 
 void
 stubs::UncachedCallHelper(VMFrame &f, uint32_t argc, bool lowered, UncachedCallResult &ucr)
 {
     ucr.init();
 
     JSContext *cx = f.cx;
     CallArgs args = CallArgsFromSp(argc, f.regs.sp);
+    RootedScript fscript(cx, f.script());
 
-    if (IsFunctionObject(args.calleev(), ucr.fun.address())) {
+    if (!ucr.setFunction(cx, args, fscript, f.pc()))
+        THROW();
+
+    if (ucr.fun) {
         if (ucr.fun->isInterpreted()) {
             InitialFrameFlags initial = lowered ? INITIAL_LOWERED : INITIAL_NONE;
             if (!UncachedInlineCall(f, initial, &ucr.codeAddr, &ucr.unjittable, argc))
                 THROW();
             return;
         }
 
         if (ucr.fun->isNative()) {
@@ -469,17 +502,16 @@ stubs::UncachedCallHelper(VMFrame &f, ui
             types::TypeScript::Monitor(f.cx, fscript, f.pc(), args.rval());
             return;
         }
     }
 
     if (!InvokeKernel(f.cx, args))
         THROW();
 
-    RootedScript fscript(cx, f.script());
     types::TypeScript::Monitor(f.cx, fscript, f.pc(), args.rval());
     return;
 }
 
 static void
 RemoveOrphanedNative(JSContext *cx, StackFrame *fp)
 {
     /*
--- a/js/src/methodjit/MonoIC.cpp
+++ b/js/src/methodjit/MonoIC.cpp
@@ -1024,23 +1024,23 @@ class CallCompiler : public BaseCompiler
         if (!linker.verifyRange(f.chunk())) {
             disable();
             return true;
         }
 
         linker.link(claspGuard, ic.slowPathStart);
         linker.link(funGuard, ic.slowPathStart);
         linker.link(done, ic.funGuard.labelAtOffset(ic.hotPathOffset));
-        JSC::CodeLocationLabel cs = linker.finalize(f);
+        ic.funJumpTarget = linker.finalize(f);
 
         JaegerSpew(JSpew_PICs, "generated CALL closure stub %p (%lu bytes)\n",
-                   cs.executableAddress(), (unsigned long) masm.size());
+                   ic.funJumpTarget.executableAddress(), (unsigned long) masm.size());
 
         Repatcher repatch(f.chunk());
-        repatch.relink(ic.funJump, cs);
+        repatch.relink(ic.funJump, ic.funJumpTarget);
 
         return true;
     }
 
     bool generateNativeStub()
     {
         /* Snapshot the frameDepth before SplatApplyArgs modifies it. */
         unsigned initialFrameDepth = f.regs.sp - f.fp()->slots();
@@ -1204,20 +1204,77 @@ class CallCompiler : public BaseCompiler
             return true;
         }
 
         linker.patchJump(ic.nativeRejoin());
 
         ic.fastGuardedNative = fun;
 
         linker.link(funGuard, ic.slowPathStart);
+        ic.funJumpTarget = linker.finalize(f);
+
+        JaegerSpew(JSpew_PICs, "generated native CALL stub %p (%lu bytes)\n",
+                   ic.funJumpTarget.executableAddress(), (unsigned long) masm.size());
+
+        Repatcher repatch(f.chunk());
+        repatch.relink(ic.funJump, ic.funJumpTarget);
+
+        return true;
+    }
+
+    bool generateCallsiteCloneStub(HandleFunction original, HandleFunction fun)
+    {
+        AutoAssertNoGC nogc;
+
+        Assembler masm;
+
+        // If we have a callsite clone, we do the folowing hack:
+        //
+        //  1) Patch funJump to a stub which guards on the identity of the
+        //     original function. If this guard fails, we jump to the original
+        //     funJump target.
+        //  2) Load the clone into the callee register.
+        //  3) Jump *back* to funGuard.
+        //
+        // For the inline path, hopefully we will succeed upon jumping back to
+        // funGuard after loading the clone.
+        //
+        // This hack is not ideal, as we can actually fail the first funGuard
+        // twice: we fail the first funGuard, pass the second funGuard, load
+        // the clone, and jump back to the first funGuard. We fail the first
+        // funGuard again, and this time we also fail the second funGuard,
+        // since a function's clone is never equal to itself. Finally, we jump
+        // to the original funJump target.
+
+        // Guard on the original function's identity.
+        Jump originalGuard = masm.branchPtr(Assembler::NotEqual, ic.funObjReg, ImmPtr(original));
+
+        // Load the clone.
+        masm.move(ImmPtr(fun), ic.funObjReg);
+
+        // Jump back to the first fun guard.
+        Jump done = masm.jump();
+
+        LinkerHelper linker(masm, JSC::JAEGER_CODE);
+        JSC::ExecutablePool *ep = linker.init(f.cx);
+        if (!ep)
+            return false;
+
+        if (!linker.verifyRange(f.chunk())) {
+            disable();
+            return true;
+        }
+
+        linker.link(originalGuard, !!ic.funJumpTarget ? ic.funJumpTarget : ic.slowPathStart);
+        linker.link(done, ic.funGuardLabel);
         JSC::CodeLocationLabel start = linker.finalize(f);
 
-        JaegerSpew(JSpew_PICs, "generated native CALL stub %p (%lu bytes)\n",
+        JaegerSpew(JSpew_PICs, "generated CALL clone stub %p (%lu bytes)\n",
                    start.executableAddress(), (unsigned long) masm.size());
+        JaegerSpew(JSpew_PICs, "guarding %p with clone %p\n", original.get(), fun.get());
 
         Repatcher repatch(f.chunk());
         repatch.relink(ic.funJump, start);
 
         return true;
     }
 
     void *update()
@@ -1298,16 +1355,19 @@ class CallCompiler : public BaseCompiler
                 if (!generateStubForClosures(fun))
                     THROWV(NULL);
             } else {
                 if (!generateFullCallStub(script, flags))
                     THROWV(NULL);
             }
         }
 
+        if (ucr.original && !generateCallsiteCloneStub(ucr.original, ucr.fun))
+            THROWV(NULL);
+
         return ucr.codeAddr;
     }
 };
 
 } // namespace mjit
 } // namespace js
 
 void * JS_FASTCALL
--- a/js/src/methodjit/MonoIC.h
+++ b/js/src/methodjit/MonoIC.h
@@ -160,26 +160,35 @@ struct CallICInfo {
     JSObject *fastGuardedObject;
     JSObject *fastGuardedNative;
 
     /* Return site for scripted calls at this site, with PC and inlining state. */
     CallSite *call;
 
     FrameSize frameSize;
 
+    /* Label to the function object identity guard. */
+    JSC::CodeLocationLabel funGuardLabel;
+
     /* Function object identity guard. */
     JSC::CodeLocationDataLabelPtr funGuard;
 
     /* Starting point for all slow call paths. */
     JSC::CodeLocationLabel slowPathStart;
 
     /* Inline to OOL jump, redirected by stubs. */
     JSC::CodeLocationJump funJump;
 
     /*
+     * Target of the above jump, remembered so that if we need to generate a
+     * callsite clone stub we can redirect to the original funJump target.
+     */
+    JSC::CodeLocationLabel funJumpTarget;
+
+    /*
      * If an Ion stub has been generated, its guard may be linked to another
      * stub. The guard location is stored in this label.
      */
     bool hasIonStub_;
     JSC::JITCode lastOolCode_;
     JSC::CodeLocationJump lastOolJump_;
 
     /* Offset to inline scripted call, from funGuard. */
--- a/js/src/methodjit/StubCalls-inl.h
+++ b/js/src/methodjit/StubCalls-inl.h
@@ -24,13 +24,34 @@ ThrowException(VMFrame &f)
 static inline void
 ReportAtomNotDefined(JSContext *cx, JSAtom *atom)
 {
     JSAutoByteString printable;
     if (js_AtomToPrintableString(cx, atom, &printable))
         js_ReportIsNotDefined(cx, printable.ptr());
 }
 
+inline bool
+stubs::UncachedCallResult::setFunction(JSContext *cx, CallArgs &args,
+                                       HandleScript callScript, jsbytecode *callPc)
+{
+    if (!IsFunctionObject(args.calleev(), fun.address()))
+        return true;
+
+    if (fun->isInterpretedLazy() && !JSFunction::getOrCreateScript(cx, fun))
+        return false;
+
+    if (cx->typeInferenceEnabled() && fun->isCloneAtCallsite()) {
+        original = fun;
+        fun = CloneFunctionAtCallsite(cx, original, callScript, callPc);
+        if (!fun)
+            return false;
+        args.setCallee(ObjectValue(*fun));
+    }
+
+    return true;
+}
+
 } /* namespace mjit */
 } /* namespace js */
 
 #endif /* jslogic_h__ */
 
--- a/js/src/methodjit/StubCalls.h
+++ b/js/src/methodjit/StubCalls.h
@@ -3,16 +3,17 @@
  *
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #if !defined jslogic_h__ && defined JS_METHODJIT
 #define jslogic_h__
 
+#include "jsfuninlines.h"
 #include "MethodJIT.h"
 
 namespace js {
 namespace mjit {
 namespace stubs {
 
 typedef enum JSTrapType {
     JSTRAP_NONE = 0,
@@ -53,26 +54,31 @@ void JS_FASTCALL ScriptProbeOnlyEpilogue
  *   (1) The function was executed in the interpreter. Then all fields
  *       are NULL except unjittable.
  *
  *   (2) The function was not executed, and the function has been compiled
  *       to JM native code. Then all fields are non-NULL.
  */
 struct UncachedCallResult {
     RootedFunction fun;        // callee function
+    RootedFunction original;   // NULL if fun is not a callsite clone, else
+                               // points to the original function.
     void           *codeAddr;  // code address of compiled callee function
     bool           unjittable; // did we try to JIT and fail?
 
-    UncachedCallResult(JSContext *cx) : fun(cx) {}
+    UncachedCallResult(JSContext *cx) : fun(cx), original(cx) {}
 
     void init() {
         fun = NULL;
+        original = NULL;
         codeAddr = NULL;
         unjittable = false;
     }
+    inline bool setFunction(JSContext *cx, CallArgs &args,
+                            HandleScript callScript, jsbytecode *callPc);
 };
 
 /*
  * Helper functions for stubs and IC functions for calling functions.
  * These functions either execute the function, return a native code
  * pointer that can be used to call the function, or throw.
  */
 void UncachedCallHelper(VMFrame &f, uint32_t argc, bool lowered, UncachedCallResult &ucr);