Bug 797131 part 3 - Add fast path for calling into Ion. r=dvander
authorJan de Mooij <jdemooij@mozilla.com>
Sat, 06 Oct 2012 11:38:18 +0200
changeset 109524 20fe6e539f7f3fbd27f6a6737acc44ca0e066347
parent 109523 edfcc6ffb63e515b7a3f02c6d00559f1f4f5156a
child 109525 e1ac3154b40e7758779061c3ff8c9bf873f954db
push id1
push userroot
push dateMon, 20 Oct 2014 17:29:22 +0000
reviewersdvander
bugs797131
milestone18.0a1
Bug 797131 part 3 - Add fast path for calling into Ion. r=dvander
js/src/ion/Bailouts.cpp
js/src/ion/Ion.cpp
js/src/ion/Ion.h
js/src/ion/IonCompartment.h
js/src/ion/IonFrames.cpp
js/src/jsinterpinlines.h
js/src/vm/Stack.cpp
--- a/js/src/ion/Bailouts.cpp
+++ b/js/src/ion/Bailouts.cpp
@@ -242,26 +242,27 @@ ConvertFrames(JSContext *cx, IonActivati
     // frames. We cannot iterate on the stack because we have no exit frame to
     // start Ion frames iteratons.
     BailoutClosure *br = js_new<BailoutClosure>();
     if (!br)
         return BAILOUT_RETURN_FATAL_ERROR;
     activation->setBailout(br);
 
     StackFrame *fp;
-    if (it.isEntryJSFrame() && cx->fp()->runningInIon()) {
+    if (it.isEntryJSFrame() && cx->fp()->runningInIon() && activation->entryfp()) {
         // Avoid creating duplicate interpreter frames. This is necessary to
         // avoid blowing out the interpreter stack, and must be used in
         // conjunction with inline-OSR from within bailouts (since each Ion
         // activation must be tied to a unique JSStackFrame for StackIter to
         // work).
         // 
         // Note: If the entry frame is a placeholder (a stub frame pushed for
         // JM -> Ion calls), then we cannot re-use it as it does not have
         // enough slots.
+        JS_ASSERT(cx->fp() == activation->entryfp());
         fp = cx->fp();
         cx->regs().sp = fp->base();
     } else {
         br->constructFrame();
         if (!cx->stack.pushBailoutArgs(cx, it, br->argsGuard()))
             return BAILOUT_RETURN_FATAL_ERROR;
 
         fp = cx->stack.pushBailoutFrame(cx, it, *br->argsGuard(), br->frameGuard());
@@ -405,22 +406,24 @@ ion::InvalidationBailout(InvalidationBai
     }
 
     iter.ionScript()->decref(cx->runtime->defaultFreeOp());
 
     if (cx->runtime->hasIonReturnOverride())
         cx->regs().sp[-1] = cx->runtime->takeIonReturnOverride();
 
     if (retval != BAILOUT_RETURN_FATAL_ERROR) {
-        if (void *annotation = activation->entryfp()->annotation()) {
-            // If the entry frame has an annotation, then we invalidated and have
-            // immediately returned into this bailout. Transfer the annotation to
-            // the new topmost frame.
-            activation->entryfp()->setAnnotation(NULL);
-            cx->fp()->setAnnotation(annotation);
+        if (activation->entryfp()) {
+            if (void *annotation = activation->entryfp()->annotation()) {
+                // If the entry frame has an annotation, then we invalidated and have
+                // immediately returned into this bailout. Transfer the annotation to
+                // the new topmost frame.
+                activation->entryfp()->setAnnotation(NULL);
+                cx->fp()->setAnnotation(annotation);
+            }
         }
 
         // If invalidation was triggered inside a stub call, we may still have to
         // monitor the result, since the bailout happens before the MMonitorTypes
         // instruction is executed.
         jsbytecode *pc = activation->bailout()->bailoutPc();
 
         // If this is not a ResumeAfter bailout, there's nothing to monitor,
--- a/js/src/ion/Ion.cpp
+++ b/js/src/ion/Ion.cpp
@@ -1430,16 +1430,73 @@ ion::SideCannon(JSContext *cx, StackFram
                  TraceLogging::ION_SIDE_CANNON_STOP,
                  script);
     }
 #endif
 
     return status;
 }
 
+IonExecStatus
+ion::FastInvoke(JSContext *cx, HandleFunction fun, CallArgs &args)
+{
+    JS_CHECK_RECURSION(cx, return IonExec_Error);
+
+    HandleScript script = fun->script();
+    IonScript *ion = script->ionScript();
+    IonCode *code = ion->method();
+    void *jitcode = code->raw();
+
+    JS_ASSERT(ion::IsEnabled(cx));
+    JS_ASSERT(!script->ion->bailoutExpected());
+
+    bool clearCallingIntoIon = false;
+    StackFrame *fp = cx->fp();
+
+    // Two cases we have to handle:
+    //
+    // (1) fp does not begin an Ion activation. This works exactly
+    //     like invoking Ion from JM: entryfp is set to fp and fp
+    //     has the callingIntoIon flag set.
+    //
+    // (2) fp already begins another IonActivation, for instance:
+    //        JM -> Ion -> array_sort -> Ion
+    //     In this cas we use an IonActivation with entryfp == NULL
+    //     and prevpc != NULL.
+    if (!fp->beginsIonActivation()) {
+        fp->setCallingIntoIon();
+        clearCallingIntoIon = true;
+        cx->runtime->ionActivation->setEntryFp(fp);
+    } else {
+        JS_ASSERT(!cx->runtime->ionActivation->entryfp());
+    }
+
+    cx->runtime->ionActivation->setPrevPc(cx->regs().pc);
+
+    EnterIonCode enter = cx->compartment->ionCompartment()->enterJITInfallible();
+    void *calleeToken = CalleeToToken(fun);
+
+    Value result = Int32Value(fun->nargs);
+    enter(jitcode, args.length() + 1, &args[0] - 1, fp, calleeToken, &result);
+
+    if (clearCallingIntoIon)
+        fp->clearCallingIntoIon();
+
+    cx->runtime->ionActivation->setEntryFp(NULL);
+    cx->runtime->ionActivation->setPrevPc(NULL);
+
+    JS_ASSERT(fp == cx->fp());
+    JS_ASSERT(!cx->runtime->hasIonReturnOverride());
+
+    args.rval().set(result);
+
+    JS_ASSERT_IF(result.isMagic(), result.isMagic(JS_ION_ERROR));
+    return result.isMagic() ? IonExec_Error : IonExec_Ok;
+}
+
 static void
 InvalidateActivation(FreeOp *fop, uint8 *ionTop, bool invalidateAll)
 {
     IonSpew(IonSpew_Invalidate, "BEGIN invalidating activation");
 
     size_t frameno = 1;
 
     for (IonFrameIterator it(ionTop); !it.done(); ++it, ++frameno) {
--- a/js/src/ion/Ion.h
+++ b/js/src/ion/Ion.h
@@ -232,16 +232,19 @@ enum IonExecStatus
     IonExec_Error,
     IonExec_Ok,
     IonExec_Bailout
 };
 
 IonExecStatus Cannon(JSContext *cx, StackFrame *fp);
 IonExecStatus SideCannon(JSContext *cx, StackFrame *fp, jsbytecode *pc);
 
+// Used to enter Ion from C++ natives like Array.map. Called from FastInvokeGuard.
+IonExecStatus FastInvoke(JSContext *cx, HandleFunction fun, CallArgs &args);
+
 // Walk the stack and invalidate active Ion frames for the invalid scripts.
 void Invalidate(types::TypeCompartment &types, FreeOp *fop,
                 const Vector<types::RecompileInfo> &invalid, bool resetUses = true);
 void Invalidate(JSContext *cx, const Vector<types::RecompileInfo> &invalid, bool resetUses = true);
 bool Invalidate(JSContext *cx, JSScript *script, bool resetUses = true);
 
 void MarkFromIon(JSCompartment *comp, Value *vp);
 
--- a/js/src/ion/IonCompartment.h
+++ b/js/src/ion/IonCompartment.h
@@ -191,19 +191,27 @@ class IonActivation
     }
     IonActivation *prev() const {
         return prev_;
     }
     uint8 *prevIonTop() const {
         return prevIonTop_;
     }
     jsbytecode *prevpc() const {
-        JS_ASSERT(entryfp_->callingIntoIon());
+        JS_ASSERT_IF(entryfp_, entryfp_->callingIntoIon());
         return prevpc_;
     }
+    void setEntryFp(StackFrame *fp) {
+        JS_ASSERT_IF(fp, !entryfp_);
+        entryfp_ = fp;
+    }
+    void setPrevPc(jsbytecode *pc) {
+        JS_ASSERT_IF(pc, !prevpc_);
+        prevpc_ = pc;
+    }
     void setBailout(BailoutClosure *bailout) {
         JS_ASSERT(!bailout_);
         bailout_ = bailout;
     }
     BailoutClosure *maybeTakeBailout() {
         BailoutClosure *br = bailout_;
         bailout_ = NULL;
         return br;
@@ -214,16 +222,22 @@ class IonActivation
     }
     BailoutClosure *bailout() const {
         JS_ASSERT(bailout_);
         return bailout_;
     }
     JSCompartment *compartment() const {
         return compartment_;
     }
+    bool empty() const {
+        // If we have an entryfp, this activation is active. However, if
+        // FastInvoke is used, entryfp may be NULL and a non-NULL prevpc
+        // indicates this activation is not empty.
+        return !entryfp_ && !prevpc_;
+    }
 
     static inline size_t offsetOfPrevPc() {
         return offsetof(IonActivation, prevpc_);
     }
     static inline size_t offsetOfEntryFp() {
         return offsetof(IonActivation, entryfp_);
     }
 };
--- a/js/src/ion/IonFrames.cpp
+++ b/js/src/ion/IonFrames.cpp
@@ -354,17 +354,17 @@ ion::HandleException(ResumeFromException
         cx->runtime->takeIonReturnOverride();
 
     rfe->stackPointer = iter.fp();
 }
 
 void
 IonActivationIterator::settle()
 {
-    while (activation_ && !activation_->entryfp()) {
+    while (activation_ && activation_->empty()) {
         top_ = activation_->prevIonTop();
         activation_ = activation_->prev();
     }
 }
 
 IonActivationIterator::IonActivationIterator(JSContext *cx)
   : top_(cx->runtime->ionTop),
     activation_(cx->runtime->ionActivation)
@@ -1011,17 +1011,24 @@ IonFrameIterator::isConstructing() const
 
         JS_ASSERT(js_CodeSpec[*inlinedParent.pc()].format & JOF_INVOKE);
 
         return (JSOp)*inlinedParent.pc() == JSOP_NEW;
     }
 
     JS_ASSERT(parent.done());
 
-    // JM ICs do not inline Ion constructor calls.
+    // If entryfp is not set, we entered Ion via a C++ native, like Array.map,
+    // using FastInvoke. FastInvoke is never used for constructor calls.
+    if (!activation_->entryfp())
+        return false;
+
+    // If callingIntoIon, we either entered Ion from JM or entered Ion from
+    // a C++ native using FastInvoke. In both of these cases we don't handle
+    // constructor calls.
     if (activation_->entryfp()->callingIntoIon())
         return false;
     JS_ASSERT(activation_->entryfp()->runningInIon());
     return activation_->entryfp()->isConstructing();
 }
 
 JSObject *
 InlineFrameIterator::thisObject() const
--- a/js/src/jsinterpinlines.h
+++ b/js/src/jsinterpinlines.h
@@ -22,16 +22,17 @@
 #include "jsatominlines.h"
 #include "jsfuninlines.h"
 #include "jsinferinlines.h"
 #include "jsopcodeinlines.h"
 #include "jspropertycacheinlines.h"
 #include "jstypedarrayinlines.h"
 
 #include "ion/Ion.h"
+#include "ion/IonCompartment.h"
 
 #include "vm/Stack-inl.h"
 
 namespace js {
 
 /*
  * Compute the implicit |this| parameter for a call expression where the callee
  * funval was resolved from an unqualified name reference to a property on obj
@@ -986,39 +987,72 @@ ReportIfNotFunction(JSContext *cx, const
  * in C++, for instance Array.map. If the callee is not Ion-compiled, this will
  * just call Invoke. If the callee has a valid IonScript, however, it will enter
  * Ion directly.
  */
 class FastInvokeGuard
 {
     InvokeArgsGuard args_;
     RootedFunction fun_;
+    RootedScript script_;
+#ifdef JS_ION
+    ion::IonContext ictx_;
+    ion::IonActivation activation_;
     bool useIon_;
+#endif
 
   public:
     FastInvokeGuard(JSContext *cx, const Value &fval)
       : fun_(cx),
+        script_(cx)
+#ifdef JS_ION
+        , ictx_(cx, cx->compartment, NULL),
+        activation_(cx, NULL),
         useIon_(ion::IsEnabled(cx))
+#endif
     {
         initFunction(fval);
     }
 
     void initFunction(const Value &fval) {
         if (fval.isObject() && fval.toObject().isFunction()) {
             JSFunction *fun = fval.toObject().toFunction();
-            if (fun->isInterpreted())
+            if (fun->isInterpreted()) {
                 fun_ = fun;
+                script_ = fun->script();
+            }
         }
     }
 
     InvokeArgsGuard &args() {
         return args_;
     }
 
     bool invoke(JSContext *cx) {
+#ifdef JS_ION
+        if (useIon_ && fun_) {
+            JS_ASSERT(fun_->script() == script_);
+
+            if (script_->hasIonScript() && !script_->ion->bailoutExpected()) {
+                ion::IonExecStatus result = ion::FastInvoke(cx, fun_, args_);
+                if (result == ion::IonExec_Error)
+                    return false;
+
+                JS_ASSERT(result == ion::IonExec_Ok);
+                return true;
+            }
+
+            if (script_->canIonCompile()) {
+                // This script is not yet hot. Since calling into Ion is much
+                // faster here, bump the use count a bit to account for this.
+                script_->incUseCount(5);
+            }
+        }
+#endif
+
         return Invoke(cx, args_);
     }
 
   private:
     FastInvokeGuard(const FastInvokeGuard& other) MOZ_DELETE;
     const FastInvokeGuard& operator=(const FastInvokeGuard& other) MOZ_DELETE;
 };
 
--- a/js/src/vm/Stack.cpp
+++ b/js/src/vm/Stack.cpp
@@ -1456,17 +1456,31 @@ StackIter::popIonFrame()
         ++ionFrames_;
         while (!ionFrames_.done() && !ionFrames_.isScripted())
             ++ionFrames_;
 
         if (!ionFrames_.done()) {
             ionInlineFrames_ = ion::InlineFrameIterator(&ionFrames_);
             pc_ = ionInlineFrames_.pc();
             script_ = ionInlineFrames_.script();
-        } else if (fp_->runningInIon()) {
+            return;
+        }
+
+        // The activation has no other frames. If entryfp is NULL, it was invoked
+        // by a native written in C++, using FastInvoke, on top of another activation.
+        ion::IonActivation *activation = ionActivations_.activation();
+        if (!activation->entryfp()) {
+            JS_ASSERT(activation->prevpc());
+            JS_ASSERT(fp_->beginsIonActivation());
+            ++ionActivations_;
+            settleOnNewState();
+            return;
+        }
+
+        if (fp_->runningInIon()) {
             ++ionActivations_;
             popFrame();
             settleOnNewState();
         } else {
             JS_ASSERT(fp_->callingIntoIon());
             state_ = SCRIPTED;
             script_ = fp_->script();
             pc_ = ionActivations_.activation()->prevpc();