author | Kannan Vijayan <kvijayan@mozilla.com> |
Thu, 15 Jan 2015 20:11:21 -0500 | |
changeset 224224 | 70a8168c7d24fafac7b10f42069d257519999bd4 |
parent 224223 | d7d0b5c0c55e356e3b6eebbd9deea1b677eb3a0a |
child 224225 | acb40b02b52de2cdea2b43bdc98d47cbc5562147 |
push id | 54160 |
push user | kvijayan@mozilla.com |
push date | Fri, 16 Jan 2015 16:05:00 +0000 |
treeherder | mozilla-inbound@809520c9cb0a [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
reviewers | jandem |
bugs | 1057082 |
milestone | 38.0a1 |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/js/src/jit/Bailouts.cpp +++ b/js/src/jit/Bailouts.cpp @@ -35,16 +35,17 @@ jit::Bailout(BailoutStack *sp, BaselineB IsInRange(FAKE_JIT_TOP_FOR_BAILOUT + sizeof(CommonFrameLayout), 0, 0x1000), "Fake jitTop pointer should be within the first page."); cx->mainThread().jitTop = FAKE_JIT_TOP_FOR_BAILOUT; JitActivationIterator jitActivations(cx->runtime()); BailoutFrameInfo bailoutData(jitActivations, sp); JitFrameIterator iter(jitActivations); MOZ_ASSERT(!iter.ionScript()->invalidated()); + CommonFrameLayout *currentFramePtr = iter.current(); TraceLoggerThread *logger = TraceLoggerForMainThread(cx->runtime()); TraceLogTimestamp(logger, TraceLogger_Bailout); JitSpew(JitSpew_IonBailouts, "Took bailout! Snapshot offset: %d", iter.snapshotOffset()); MOZ_ASSERT(IsBaselineEnabled(cx)); @@ -84,16 +85,36 @@ jit::Bailout(BailoutStack *sp, BaselineB // already in a bailout, we could not switch to an invalidation // bailout. When the code of an IonScript which is on the stack is // invalidated (see InvalidateActivation), we remove references to it and // increment the reference counter for each activation that appear on the // stack. As the bailed frame is one of them, we have to decrement it now. if (iter.ionScript()->invalidated()) iter.ionScript()->decrementInvalidationCount(cx->runtime()->defaultFreeOp()); + // NB: Commentary on how |lastProfilingFrame| is set from bailouts. + // + // Once we return to jitcode, any following frames might get clobbered, + // but the current frame will not (as it will be clobbered "in-place" + // with a baseline frame that will share the same frame prefix). + // However, there may be multiple baseline frames unpacked from this + // single Ion frame, which means we will need to once again reset + // |lastProfilingFrame| to point to the correct unpacked last frame + // in |FinishBailoutToBaseline|. + // + // In the case of error, the jitcode will jump immediately to an + // exception handler, which will unwind the frames and properly set + // the |lastProfilingFrame| to point to the frame being resumed into + // (see |AutoResetLastProfilerFrameOnReturnFromException|). + // + // In both cases, we want to temporarily set the |lastProfilingFrame| + // to the current frame being bailed out, and then fix it up later. + if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(cx->runtime())) + cx->mainThread().jitActivation->setLastProfilingFrame(currentFramePtr); + return retval; } uint32_t jit::InvalidationBailout(InvalidationBailoutStack *sp, size_t *frameSizeOut, BaselineBailoutInfo **bailoutInfo) { sp->checkInvariants(); @@ -101,16 +122,17 @@ jit::InvalidationBailout(InvalidationBai JSContext *cx = GetJSContextFromJitCode(); // We don't have an exit frame. cx->mainThread().jitTop = FAKE_JIT_TOP_FOR_BAILOUT; JitActivationIterator jitActivations(cx->runtime()); BailoutFrameInfo bailoutData(jitActivations, sp); JitFrameIterator iter(jitActivations); + CommonFrameLayout *currentFramePtr = iter.current(); TraceLoggerThread *logger = TraceLoggerForMainThread(cx->runtime()); TraceLogTimestamp(logger, TraceLogger_Invalidation); JitSpew(JitSpew_IonBailouts, "Took invalidation bailout! Snapshot offset: %d", iter.snapshotOffset()); // Note: the frame size must be computed before we return from this function. *frameSizeOut = iter.frameSize(); @@ -156,16 +178,20 @@ jit::InvalidationBailout(InvalidationBai JitSpew(JitSpew_IonInvalidate, " new calleeToken %p", (void *) frame->calleeToken()); JitSpew(JitSpew_IonInvalidate, " new frameSize %u", unsigned(frame->prevFrameLocalSize())); JitSpew(JitSpew_IonInvalidate, " new ra %p", (void *) frame->returnAddress()); } iter.ionScript()->decrementInvalidationCount(cx->runtime()->defaultFreeOp()); + // Make the frame being bailed out the top profiled frame. + if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(cx->runtime())) + cx->mainThread().jitActivation->setLastProfilingFrame(currentFramePtr); + return retval; } BailoutFrameInfo::BailoutFrameInfo(const JitActivationIterator &activations, const JitFrameIterator &frame) : machine_(frame.machineState()) { framePointer_ = (uint8_t *) frame.fp(); @@ -189,16 +215,17 @@ jit::ExceptionHandlerBailout(JSContext * MOZ_ASSERT_IF(!excInfo.propagatingIonExceptionForDebugMode(), cx->isExceptionPending()); cx->mainThread().jitTop = FAKE_JIT_TOP_FOR_BAILOUT; gc::AutoSuppressGC suppress(cx); JitActivationIterator jitActivations(cx->runtime()); BailoutFrameInfo bailoutData(jitActivations, frame.frame()); JitFrameIterator iter(jitActivations); + CommonFrameLayout *currentFramePtr = iter.current(); BaselineBailoutInfo *bailoutInfo = nullptr; uint32_t retval = BailoutIonToBaseline(cx, bailoutData.activation(), iter, true, &bailoutInfo, &excInfo, poppedLastSPSFrameOut); if (retval == BAILOUT_RETURN_OK) { MOZ_ASSERT(bailoutInfo); @@ -221,16 +248,20 @@ jit::ExceptionHandlerBailout(JSContext * cx->clearPendingException(); if (retval == BAILOUT_RETURN_OVERRECURSED) *overrecursed = true; else MOZ_ASSERT(retval == BAILOUT_RETURN_FATAL_ERROR); } + // Make the frame being bailed out the top profiled frame. + if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(cx->runtime())) + cx->mainThread().jitActivation->setLastProfilingFrame(currentFramePtr); + return retval; } // Initialize the decl env Object, call object, and any arguments obj of the current frame. bool jit::EnsureHasScopeObjects(JSContext *cx, AbstractFramePtr fp) { if (fp.isFunctionFrame() &&
--- a/js/src/jit/BaselineBailouts.cpp +++ b/js/src/jit/BaselineBailouts.cpp @@ -1716,16 +1716,22 @@ jit::FinishBailoutToBaseline(BaselineBai // that script->needsArgsObj() implies frame->hasArgsObj(). RootedScript innerScript(cx, nullptr); RootedScript outerScript(cx, nullptr); MOZ_ASSERT(cx->currentlyRunningInJit()); JitFrameIterator iter(cx); uint8_t *outerFp = nullptr; + // Iter currently points at the exit frame. Get the previous frame + // (which must be a baseline frame), and set it as the last profiling + // frame. + if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(cx->runtime())) + cx->mainThread().jitActivation->setLastProfilingFrame(iter.prevFp()); + uint32_t frameno = 0; while (frameno < numFrames) { MOZ_ASSERT(!iter.isIonJS()); if (iter.isBaselineJS()) { BaselineFrame *frame = iter.baselineFrame(); MOZ_ASSERT(frame->script()->hasBaselineScript());
--- a/js/src/jit/BaselineCompiler.cpp +++ b/js/src/jit/BaselineCompiler.cpp @@ -171,29 +171,33 @@ BaselineCompiler::compile() } if (pcEntries.oom()) return Method_Error; prologueOffset_.fixup(&masm); epilogueOffset_.fixup(&masm); spsPushToggleOffset_.fixup(&masm); + profilerEnterFrameToggleOffset_.fixup(&masm); + profilerExitFrameToggleOffset_.fixup(&masm); #ifdef JS_TRACE_LOGGING traceLoggerEnterToggleOffset_.fixup(&masm); traceLoggerExitToggleOffset_.fixup(&masm); #endif postDebugPrologueOffset_.fixup(&masm); // Note: There is an extra entry in the bytecode type map for the search hint, see below. size_t bytecodeTypeMapEntries = script->nTypeSets() + 1; mozilla::UniquePtr<BaselineScript, JS::DeletePolicy<BaselineScript> > baselineScript( BaselineScript::New(script, prologueOffset_.offset(), epilogueOffset_.offset(), spsPushToggleOffset_.offset(), + profilerEnterFrameToggleOffset_.offset(), + profilerExitFrameToggleOffset_.offset(), traceLoggerEnterToggleOffset_.offset(), traceLoggerExitToggleOffset_.offset(), postDebugPrologueOffset_.offset(), icEntries_.length(), pcMappingIndexEntries.length(), pcEntries.length(), bytecodeTypeMapEntries, yieldOffsets_.length())); @@ -258,29 +262,33 @@ BaselineCompiler::compile() // searches for the sought entry when queries are in linear order. bytecodeMap[script->nTypeSets()] = 0; baselineScript->copyYieldEntries(script, yieldOffsets_); if (compileDebugInstrumentation_) baselineScript->setHasDebugInstrumentation(); - // Register a native => bytecode mapping entry for this script if needed. - if (cx->runtime()->jitRuntime()->isNativeToBytecodeMapEnabled(cx->runtime())) { + // If profiler instrumentation is enabled, register a native => bytecode mapping entry, + // and toggle profiling on + if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(cx->runtime())) { JitSpew(JitSpew_Profiling, "Added JitcodeGlobalEntry for baseline script %s:%d (%p)", script->filename(), script->lineno(), baselineScript.get()); JitcodeGlobalEntry::BaselineEntry entry; entry.init(code->raw(), code->raw() + code->instructionsSize(), script); JitcodeGlobalTable *globalTable = cx->runtime()->jitRuntime()->getJitcodeGlobalTable(); if (!globalTable->addEntry(entry)) return Method_Error; // Mark the jitcode as having a bytecode map. code->setHasBytecodeMap(); + + // Toggle profiler instrumentation on in the jitcode. + baselineScript->toggleProfilerInstrumentation(true); } script->setBaselineScript(cx, baselineScript.release()); return Method_Compiled; } void @@ -319,16 +327,18 @@ BaselineCompiler::emitInitializeLocals(s bool BaselineCompiler::emitPrologue() { #ifdef JS_USE_LINK_REGISTER // Push link register from generateEnterJIT()'s BLR. masm.pushReturnAddress(); masm.checkStackAlignment(); #endif + emitProfilerEnterFrame(); + masm.push(BaselineFrameReg); masm.mov(BaselineStackReg, BaselineFrameReg); masm.subPtr(Imm32(BaselineFrame::Size()), BaselineStackReg); // Initialize BaselineFrame. For eval scripts, the scope chain // is passed in R1, so we have to be careful not to clobber // it. @@ -433,16 +443,18 @@ BaselineCompiler::emitEpilogue() #endif // Pop SPS frame if necessary emitSPSPop(); masm.mov(BaselineFrameReg, BaselineStackReg); masm.pop(BaselineFrameReg); + emitProfilerExitFrame(); + masm.ret(); return true; } // On input: // R2.scratchReg() contains object being written to. // Called with the baseline stack synced, except for R0 which is preserved. // All other registers are usable as scratch. @@ -851,16 +863,46 @@ BaselineCompiler::emitSPSPop() // If profiler entry was pushed on this frame, pop it. Label noPop; masm.branchTest32(Assembler::Zero, frame.addressOfFlags(), Imm32(BaselineFrame::HAS_PUSHED_SPS_FRAME), &noPop); masm.spsPopFrameSafe(&cx->runtime()->spsProfiler, R1.scratchReg()); masm.bind(&noPop); } +void +BaselineCompiler::emitProfilerEnterFrame() +{ + // Store stack position to lastProfilingFrame variable, guarded by a toggled jump. + // Starts off initially disabled. + Label noInstrument; + CodeOffsetLabel toggleOffset = masm.toggledJump(&noInstrument); + masm.profilerEnterFrame(BaselineStackReg, R0.scratchReg()); + masm.bind(&noInstrument); + + // Store the start offset in the appropriate location. + MOZ_ASSERT(profilerEnterFrameToggleOffset_.offset() == 0); + profilerEnterFrameToggleOffset_ = toggleOffset; +} + +void +BaselineCompiler::emitProfilerExitFrame() +{ + // Store previous frame to lastProfilingFrame variable, guarded by a toggled jump. + // Starts off initially disabled. + Label noInstrument; + CodeOffsetLabel toggleOffset = masm.toggledJump(&noInstrument); + masm.profilerExitFrame(); + masm.bind(&noInstrument); + + // Store the start offset in the appropriate location. + MOZ_ASSERT(profilerExitFrameToggleOffset_.offset() == 0); + profilerExitFrameToggleOffset_ = toggleOffset; +} + MethodStatus BaselineCompiler::emitBody() { MOZ_ASSERT(pc == script->code()); bool lastOpUnreachable = false; uint32_t emittedOps = 0; mozilla::DebugOnly<jsbytecode *> prevpc = pc; @@ -3625,16 +3667,29 @@ BaselineCompiler::emit_JSOP_RESUME() ICEntry icEntry(script->pcToOffset(pc), ICEntry::Kind_Op); icEntry.setReturnOffset(CodeOffsetLabel(masm.currentOffset())); if (!icEntries_.append(icEntry)) return false; masm.jump(&returnTarget); masm.bind(&genStart); + // If profiler instrumentation is on, update lastProfilingFrame on + // current JitActivation + { + Register scratchReg = scratch2; + Label skip; + AbsoluteAddress addressOfEnabled(cx->runtime()->spsProfiler.addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &skip); + masm.loadPtr(AbsoluteAddress(cx->mainThread().addressOfProfilingActivation()), scratchReg); + masm.storePtr(BaselineStackReg, + Address(scratchReg, JitActivation::offsetOfLastProfilingFrame())); + masm.bind(&skip); + } + // Construct BaselineFrame. masm.push(BaselineFrameReg); masm.mov(BaselineStackReg, BaselineFrameReg); masm.subPtr(Imm32(BaselineFrame::Size()), BaselineStackReg); masm.checkStackAlignment(); // Store flags and scope chain. masm.store32(Imm32(BaselineFrame::HAS_CALL_OBJ), frame.addressOfFlags());
--- a/js/src/jit/BaselineCompiler.h +++ b/js/src/jit/BaselineCompiler.h @@ -254,16 +254,19 @@ class BaselineCompiler : public Baseline bool emitArgumentTypeChecks(); bool emitDebugPrologue(); bool emitDebugTrap(); bool emitTraceLoggerEnter(); bool emitTraceLoggerExit(); bool emitSPSPush(); void emitSPSPop(); + void emitProfilerEnterFrame(); + void emitProfilerExitFrame(); + bool initScopeChain(); void storeValue(const StackValue *source, const Address &dest, const ValueOperand &scratch); #define EMIT_OP(op) bool emit_##op(); OPCODE_LIST(EMIT_OP) #undef EMIT_OP
--- a/js/src/jit/BaselineIC.cpp +++ b/js/src/jit/BaselineIC.cpp @@ -1073,16 +1073,31 @@ ICWarmUpCounter_Fallback::Compiler::gene // Restore the stack pointer to point to the saved frame pointer. masm.movePtr(BaselineFrameReg, BaselineStackReg); // Discard saved frame pointer, so that the return address is on top of // the stack. masm.pop(scratchReg); +#ifdef DEBUG + // If profiler instrumentation is on, ensure that lastProfilingFrame is + // the frame currently being OSR-ed + { + Label checkOk; + AbsoluteAddress addressOfEnabled(cx->runtime()->spsProfiler.addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &checkOk); + masm.loadPtr(AbsoluteAddress((void*)&cx->mainThread().jitActivation), scratchReg); + masm.loadPtr(Address(scratchReg, JitActivation::offsetOfLastProfilingFrame()), scratchReg); + masm.branchPtr(Assembler::Equal, scratchReg, BaselineStackReg, &checkOk); + masm.assumeUnreachable("Baseline OSR lastProfilingFrame mismatch."); + masm.bind(&checkOk); + } +#endif + // Jump into Ion. masm.loadPtr(Address(osrDataReg, offsetof(IonOsrTempData, jitcode)), scratchReg); masm.loadPtr(Address(osrDataReg, offsetof(IonOsrTempData, baselineFrame)), OsrFrameReg); masm.jump(scratchReg); // No jitcode available, do nothing. masm.bind(&noCompiledCode); EmitReturnFromIC(masm);
--- a/js/src/jit/BaselineJIT.cpp +++ b/js/src/jit/BaselineJIT.cpp @@ -37,29 +37,34 @@ PCMappingSlotInfo::ToSlotLocation(const MOZ_ASSERT(stackVal->reg() == R1); return SlotInR1; } MOZ_ASSERT(stackVal->kind() != StackValue::Stack); return SlotIgnore; } BaselineScript::BaselineScript(uint32_t prologueOffset, uint32_t epilogueOffset, - uint32_t spsPushToggleOffset, uint32_t traceLoggerEnterToggleOffset, + uint32_t spsPushToggleOffset, + uint32_t profilerEnterToggleOffset, + uint32_t profilerExitToggleOffset, + uint32_t traceLoggerEnterToggleOffset, uint32_t traceLoggerExitToggleOffset, uint32_t postDebugPrologueOffset) : method_(nullptr), templateScope_(nullptr), fallbackStubSpace_(), dependentAsmJSModules_(nullptr), prologueOffset_(prologueOffset), epilogueOffset_(epilogueOffset), #ifdef DEBUG spsOn_(false), #endif spsPushToggleOffset_(spsPushToggleOffset), + profilerEnterToggleOffset_(profilerEnterToggleOffset), + profilerExitToggleOffset_(profilerExitToggleOffset), #ifdef JS_TRACE_LOGGING # ifdef DEBUG traceLoggerScriptsEnabled_(false), traceLoggerEngineEnabled_(false), # endif traceLoggerEnterToggleOffset_(traceLoggerEnterToggleOffset), traceLoggerExitToggleOffset_(traceLoggerExitToggleOffset), traceLoggerScriptEvent_(), @@ -337,18 +342,20 @@ jit::CanEnterBaselineMethod(JSContext *c } RootedScript script(cx, state.script()); return CanEnterBaselineJIT(cx, script, /* osrFrame = */ nullptr); }; BaselineScript * BaselineScript::New(JSScript *jsscript, uint32_t prologueOffset, uint32_t epilogueOffset, - uint32_t spsPushToggleOffset, uint32_t traceLoggerEnterToggleOffset, - uint32_t traceLoggerExitToggleOffset, uint32_t postDebugPrologueOffset, + uint32_t spsPushToggleOffset, + uint32_t profilerEnterToggleOffset, uint32_t profilerExitToggleOffset, + uint32_t traceLoggerEnterToggleOffset, uint32_t traceLoggerExitToggleOffset, + uint32_t postDebugPrologueOffset, size_t icEntries, size_t pcMappingIndexEntries, size_t pcMappingSize, size_t bytecodeTypeMapEntries, size_t yieldEntries) { static const unsigned DataAlignment = sizeof(uintptr_t); size_t icEntriesSize = icEntries * sizeof(ICEntry); size_t pcMappingIndexEntriesSize = pcMappingIndexEntries * sizeof(PCMappingIndexEntry); size_t bytecodeTypeMapSize = bytecodeTypeMapEntries * sizeof(uint32_t); @@ -365,18 +372,20 @@ BaselineScript::New(JSScript *jsscript, paddedPCMappingSize + paddedBytecodeTypesMapSize + paddedYieldEntriesSize; BaselineScript *script = jsscript->zone()->pod_malloc_with_extra<BaselineScript, uint8_t>(allocBytes); if (!script) return nullptr; new (script) BaselineScript(prologueOffset, epilogueOffset, - spsPushToggleOffset, traceLoggerEnterToggleOffset, - traceLoggerExitToggleOffset, postDebugPrologueOffset); + spsPushToggleOffset, + profilerEnterToggleOffset, profilerExitToggleOffset, + traceLoggerEnterToggleOffset, traceLoggerExitToggleOffset, + postDebugPrologueOffset); size_t offsetCursor = sizeof(BaselineScript); MOZ_ASSERT(offsetCursor == AlignBytes(sizeof(BaselineScript), DataAlignment)); script->icEntriesOffset_ = offsetCursor; script->icEntries_ = icEntries; offsetCursor += paddedICEntriesSize; @@ -946,16 +955,39 @@ BaselineScript::toggleTraceLoggerEngine( #if DEBUG traceLoggerEngineEnabled_ = enable; #endif } #endif void +BaselineScript::toggleProfilerInstrumentation(bool enable) +{ + if (enable == isProfilerInstrumentationOn()) + return; + + JitSpew(JitSpew_BaselineIC, " toggling SPS %s for BaselineScript %p", + enable ? "on" : "off", this); + + // Toggle the jump + CodeLocationLabel enterToggleLocation(method_, CodeOffsetLabel(profilerEnterToggleOffset_)); + CodeLocationLabel exitToggleLocation(method_, CodeOffsetLabel(profilerExitToggleOffset_)); + if (enable) { + Assembler::ToggleToCmp(enterToggleLocation); + Assembler::ToggleToCmp(exitToggleLocation); + flags_ |= uint32_t(PROFILER_INSTRUMENTATION_ON); + } else { + Assembler::ToggleToJmp(enterToggleLocation); + Assembler::ToggleToJmp(exitToggleLocation); + flags_ &= ~uint32_t(PROFILER_INSTRUMENTATION_ON); + } +} + +void BaselineScript::purgeOptimizedStubs(Zone *zone) { JitSpew(JitSpew_BaselineIC, "Purging optimized stubs"); for (size_t i = 0; i < numICEntries(); i++) { ICEntry &entry = icEntry(i); if (!entry.hasStub()) continue; @@ -1048,16 +1080,17 @@ void jit::ToggleBaselineSPS(JSRuntime *runtime, bool enable) { for (ZonesIter zone(runtime, SkipAtoms); !zone.done(); zone.next()) { for (gc::ZoneCellIter i(zone, gc::FINALIZE_SCRIPT); !i.done(); i.next()) { JSScript *script = i.get<JSScript>(); if (!script->hasBaselineScript()) continue; script->baselineScript()->toggleSPS(enable); + script->baselineScript()->toggleProfilerInstrumentation(enable); } } } #ifdef JS_TRACE_LOGGING void jit::ToggleBaselineTraceLoggerScripts(JSRuntime *runtime, bool enable) {
--- a/js/src/jit/BaselineJIT.h +++ b/js/src/jit/BaselineJIT.h @@ -139,16 +139,18 @@ struct BaselineScript // returned from. uint32_t epilogueOffset_; // The offsets for the toggledJump instructions for SPS update ICs. #ifdef DEBUG mozilla::DebugOnly<bool> spsOn_; #endif uint32_t spsPushToggleOffset_; + uint32_t profilerEnterToggleOffset_; + uint32_t profilerExitToggleOffset_; // The offsets and event used for Tracelogger toggling. #ifdef JS_TRACE_LOGGING # ifdef DEBUG bool traceLoggerScriptsEnabled_; bool traceLoggerEngineEnabled_; # endif uint32_t traceLoggerEnterToggleOffset_; @@ -180,17 +182,20 @@ struct BaselineScript // Flag set when compiled for use with Debugger. Handles various // Debugger hooks and compiles toggled calls for traps. HAS_DEBUG_INSTRUMENTATION = 1 << 3, // Flag set if this script has ever been Ion compiled, either directly // or inlined into another script. This is cleared when the script's // type information or caches are cleared. - ION_COMPILED_OR_INLINED = 1 << 4 + ION_COMPILED_OR_INLINED = 1 << 4, + + // Flag is set if this script has profiling instrumentation turned on. + PROFILER_INSTRUMENTATION_ON = 1 << 5 }; private: uint32_t flags_; private: void trace(JSTracer *trc); @@ -209,24 +214,32 @@ struct BaselineScript // For generator scripts, we store the native code address for each yield // instruction. uint32_t yieldEntriesOffset_; public: // Do not call directly, use BaselineScript::New. This is public for cx->new_. BaselineScript(uint32_t prologueOffset, uint32_t epilogueOffset, - uint32_t spsPushToggleOffset, uint32_t traceLoggerEnterToggleOffset, - uint32_t traceLoggerExitToggleOffset, uint32_t postDebugPrologueOffset); + uint32_t spsPushToggleOffset, + uint32_t profilerEnterToggleOffset, + uint32_t profilerExitToggleOffset, + uint32_t traceLoggerEnterToggleOffset, + uint32_t traceLoggerExitToggleOffset, + uint32_t postDebugPrologueOffset); static BaselineScript *New(JSScript *jsscript, uint32_t prologueOffset, uint32_t epilogueOffset, uint32_t postDebugPrologueOffset, - uint32_t spsPushToggleOffset, uint32_t traceLoggerEnterToggleOffset, - uint32_t traceLoggerExitToggleOffset, size_t icEntries, - size_t pcMappingIndexEntries, size_t pcMappingSize, + uint32_t spsPushToggleOffset, + uint32_t profilerEnterToggleOffset, + uint32_t profilerExitToggleOffset, + uint32_t traceLoggerEnterToggleOffset, + uint32_t traceLoggerExitToggleOffset, + size_t icEntries, size_t pcMappingIndexEntries, + size_t pcMappingSize, size_t bytecodeTypeMapEntries, size_t yieldEntries); static void Trace(JSTracer *trc, BaselineScript *script); static void Destroy(FreeOp *fop, BaselineScript *script); void purgeOptimizedStubs(Zone *zone); static inline size_t offsetOfMethod() { @@ -381,16 +394,20 @@ struct BaselineScript void removeDependentAsmJSModule(DependentAsmJSModuleExit exit); // Toggle debug traps (used for breakpoints and step mode) in the script. // If |pc| is nullptr, toggle traps for all ops in the script. Else, only // toggle traps at |pc|. void toggleDebugTraps(JSScript *script, jsbytecode *pc); void toggleSPS(bool enable); + void toggleProfilerInstrumentation(bool enable); + bool isProfilerInstrumentationOn() const { + return flags_ & PROFILER_INSTRUMENTATION_ON; + } #ifdef JS_TRACE_LOGGING void initTraceLogger(JSRuntime *runtime, JSScript *script); void toggleTraceLoggerScripts(JSRuntime *runtime, JSScript *script, bool enable); void toggleTraceLoggerEngine(bool enable); static size_t offsetOfTraceLoggerScriptEvent() { return offsetof(BaselineScript, traceLoggerScriptEvent_);
--- a/js/src/jit/CodeGenerator.cpp +++ b/js/src/jit/CodeGenerator.cpp @@ -1976,25 +1976,31 @@ CodeGenerator::visitReturn(LReturn *lir) // Don't emit a jump to the return label if this is the last block. if (current->mir() != *gen->graph().poBegin()) masm.jump(&returnLabel_); } void CodeGenerator::visitOsrEntry(LOsrEntry *lir) { + Register temp = ToRegister(lir->temp()); + // Remember the OSR entry offset into the code buffer. masm.flushBuffer(); setOsrEntryOffset(masm.size()); #ifdef JS_TRACE_LOGGING emitTracelogStopEvent(TraceLogger_Baseline); emitTracelogStartEvent(TraceLogger_IonMonkey); #endif + // If profiling, save the current frame pointer to a per-thread global field. + if (isProfilerInstrumentationEnabled()) + masm.profilerEnterFrame(StackPointer, temp); + // Allocate the full frame for this function // Note we have a new entry here. So we reset MacroAssembler::framePushed() // to 0, before reserving the stack. MOZ_ASSERT(masm.framePushed() == frameSize()); masm.setFramePushed(0); masm.reserveStack(frameSize()); } @@ -7149,17 +7155,17 @@ CodeGenerator::link(JSContext *cx, types // read barriers which were skipped while compiling the script off thread. Linker linker(masm); AutoFlushICache afc("IonLink"); JitCode *code = linker.newCodeForIonScript(cx); if (!code) return false; // Encode native to bytecode map if profiling is enabled. - if (isNativeToBytecodeMapEnabled()) { + if (isProfilerInstrumentationEnabled()) { // Generate native-to-bytecode main table. if (!generateCompactNativeToBytecodeMap(cx, code)) return false; uint8_t *ionTableAddr = ((uint8_t *) nativeToBytecodeMap_) + nativeToBytecodeTableOffset_; JitcodeIonTable *ionTable = (JitcodeIonTable *) ionTableAddr; // Construct the IonEntry that will go into the global table.
--- a/js/src/jit/IonCaches.cpp +++ b/js/src/jit/IonCaches.cpp @@ -424,17 +424,17 @@ IonCache::linkAndAttachStub(JSContext *c #ifdef JS_ION_PERF writePerfSpewerJitCodeProfile(code, "IonCache"); #endif attachStub(masm, attacher, code); // Add entry to native => bytecode mapping for this stub if needed. - if (cx->runtime()->jitRuntime()->isNativeToBytecodeMapEnabled(cx->runtime())) { + if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(cx->runtime())) { JitcodeGlobalEntry::IonCacheEntry entry; entry.init(code->raw(), code->raw() + code->instructionsSize(), rejoinAddress()); // Add entry to the global table. JitcodeGlobalTable *globalTable = cx->runtime()->jitRuntime()->getJitcodeGlobalTable(); if (!globalTable->addEntry(entry)) { entry.destroy(); return false;
--- a/js/src/jit/JitCompartment.h +++ b/js/src/jit/JitCompartment.h @@ -394,22 +394,18 @@ class JitRuntime return jitcodeGlobalTable_ != nullptr; } JitcodeGlobalTable *getJitcodeGlobalTable() { MOZ_ASSERT(hasJitcodeGlobalTable()); return jitcodeGlobalTable_; } - bool isNativeToBytecodeMapEnabled(JSRuntime *rt) { -#ifdef DEBUG - return true; -#else // DEBUG + bool isProfilerInstrumentationEnabled(JSRuntime *rt) { return rt->spsProfiler.enabled(); -#endif // DEBUG } }; class JitZone { // Allocated space for optimized baseline stubs. OptimizedICStubSpace optimizedStubSpace_;
--- a/js/src/jit/JitFrames.cpp +++ b/js/src/jit/JitFrames.cpp @@ -24,16 +24,17 @@ #include "jit/PcScriptCache.h" #include "jit/Recover.h" #include "jit/Safepoints.h" #include "jit/Snapshots.h" #include "jit/VMFunctions.h" #include "vm/ArgumentsObject.h" #include "vm/Debugger.h" #include "vm/Interpreter.h" +#include "vm/SPSProfiler.h" #include "vm/TraceLogging.h" #include "jsinferinlines.h" #include "jsscriptinlines.h" #include "gc/Nursery-inl.h" #include "jit/JitFrameIterator-inl.h" #include "vm/Debugger-inl.h" #include "vm/Probes-inl.h" @@ -677,22 +678,64 @@ struct AutoDeleteDebugModeOSRInfo struct AutoClearBaselineOverridePc { BaselineFrame *frame; explicit AutoClearBaselineOverridePc(BaselineFrame *frame) : frame(frame) { MOZ_ASSERT(frame); } ~AutoClearBaselineOverridePc() { frame->clearOverridePc(); } }; +struct AutoResetLastProfilerFrameOnReturnFromException +{ + JSContext *cx; + ResumeFromException *rfe; + + AutoResetLastProfilerFrameOnReturnFromException(JSContext *cx, ResumeFromException *rfe) + : cx(cx), rfe(rfe) {} + + ~AutoResetLastProfilerFrameOnReturnFromException() { + if (!cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(cx->runtime())) + return; + + MOZ_ASSERT(cx->mainThread().jitActivation == cx->mainThread().profilingActivation()); + + void *lastProfilingFrame = getLastProfilingFrame(); + cx->mainThread().jitActivation->setLastProfilingFrame(lastProfilingFrame); + } + + void *getLastProfilingFrame() { + switch (rfe->kind) { + case ResumeFromException::RESUME_ENTRY_FRAME: + return nullptr; + + // The following all return into baseline frames. + case ResumeFromException::RESUME_CATCH: + case ResumeFromException::RESUME_FINALLY: + case ResumeFromException::RESUME_FORCED_RETURN: + return rfe->framePointer + BaselineFrame::FramePointerOffset; + + // When resuming into a bailed-out ion frame, use the bailout info to + // find the frame we are resuming into. + case ResumeFromException::RESUME_BAILOUT: + return rfe->bailoutInfo->incomingStack; + } + + MOZ_CRASH("Invalid ResumeFromException type!"); + return nullptr; + } +}; + void HandleException(ResumeFromException *rfe) { JSContext *cx = GetJSContextFromJitCode(); TraceLoggerThread *logger = TraceLoggerForMainThread(cx->runtime()); + AutoResetLastProfilerFrameOnReturnFromException profFrameReset(cx, rfe); + rfe->kind = ResumeFromException::RESUME_ENTRY_FRAME; JitSpew(JitSpew_IonInvalidate, "handling exception"); // Clear any Ion return override that's been set. // This may happen if a callVM function causes an invalidation (setting the // override), and then fails, bypassing the bailout handlers that would // otherwise clear the return override.
--- a/js/src/jit/JitFrames.h +++ b/js/src/jit/JitFrames.h @@ -326,16 +326,19 @@ class CommonFrameLayout uintptr_t descriptor_; static const uintptr_t FrameTypeMask = (1 << FRAMETYPE_BITS) - 1; public: static size_t offsetOfDescriptor() { return offsetof(CommonFrameLayout, descriptor_); } + uintptr_t descriptor() const { + return descriptor_; + } static size_t offsetOfReturnAddress() { return offsetof(CommonFrameLayout, returnAddress_); } FrameType prevType() const { return FrameType(descriptor_ & FrameTypeMask); } void changePrevType(FrameType type) { descriptor_ &= ~FrameTypeMask; @@ -818,16 +821,21 @@ class BaselineStubFrameLayout : public C static inline int reverseOffsetOfStubPtr() { return -int(sizeof(void *)); } static inline int reverseOffsetOfSavedFramePtr() { return -int(2 * sizeof(void *)); } + void *reverseSavedFramePtr() { + uint8_t *addr = ((uint8_t *) this) + reverseOffsetOfSavedFramePtr(); + return *(void **)addr; + } + inline ICStub *maybeStubPtr() { uint8_t *fp = reinterpret_cast<uint8_t *>(this); return *reinterpret_cast<ICStub **>(fp + reverseOffsetOfStubPtr()); } inline void setStubPtr(ICStub *stub) { uint8_t *fp = reinterpret_cast<uint8_t *>(this); *reinterpret_cast<ICStub **>(fp + reverseOffsetOfStubPtr()) = stub; }
--- a/js/src/jit/LIR-Common.h +++ b/js/src/jit/LIR-Common.h @@ -3661,39 +3661,43 @@ class LFloat32x4ToInt32x4 : public LInst class LStart : public LInstructionHelper<0, 0, 0> { public: LIR_HEADER(Start) }; // Passed the BaselineFrame address in the OsrFrameReg by SideCannon(). // Forwards this object to the LOsrValues for Value materialization. -class LOsrEntry : public LInstructionHelper<1, 0, 0> +class LOsrEntry : public LInstructionHelper<1, 0, 1> { protected: Label label_; uint32_t frameDepth_; public: LIR_HEADER(OsrEntry) - LOsrEntry() + LOsrEntry(const LDefinition &temp) : frameDepth_(0) - { } + { + setTemp(0, temp); + } void setFrameDepth(uint32_t depth) { frameDepth_ = depth; } uint32_t getFrameDepth() { return frameDepth_; } Label *label() { return &label_; } - + const LDefinition *temp() { + return getTemp(0); + } }; // Materialize a Value stored in an interpreter frame for OSR. class LOsrValue : public LInstructionHelper<BOX_PIECES, 1, 0> { public: LIR_HEADER(OsrValue)
--- a/js/src/jit/Lowering.cpp +++ b/js/src/jit/Lowering.cpp @@ -1628,17 +1628,17 @@ void LIRGenerator::visitLimitedTruncate(MLimitedTruncate *nop) { redefine(nop, nop->input()); } void LIRGenerator::visitOsrEntry(MOsrEntry *entry) { - LOsrEntry *lir = new(alloc()) LOsrEntry; + LOsrEntry *lir = new(alloc()) LOsrEntry(temp()); defineFixed(lir, entry, LAllocation(AnyRegister(OsrFrameReg))); } void LIRGenerator::visitOsrValue(MOsrValue *value) { LOsrValue *lir = new(alloc()) LOsrValue(useRegister(value->entry())); defineBox(lir, value);
--- a/js/src/jit/MIRGenerator.h +++ b/js/src/jit/MIRGenerator.h @@ -78,24 +78,18 @@ class MIRGenerator bool instrumentedProfiling() { if (!instrumentedProfilingIsCached_) { instrumentedProfiling_ = GetJitContext()->runtime->spsProfiler().enabled(); instrumentedProfilingIsCached_ = true; } return instrumentedProfiling_; } - bool isNativeToBytecodeMapEnabled() { - if (compilingAsmJS()) - return false; -#ifdef DEBUG - return true; -#else - return instrumentedProfiling(); -#endif + bool isProfilerInstrumentationEnabled() { + return !compilingAsmJS() && instrumentedProfiling(); } // Whether the main thread is trying to cancel this build. bool shouldCancel(const char *why) { maybePause(); return cancelBuild_; } void cancel() {
--- a/js/src/jit/arm/CodeGenerator-arm.cpp +++ b/js/src/jit/arm/CodeGenerator-arm.cpp @@ -42,16 +42,21 @@ CodeGeneratorARM::CodeGeneratorARM(MIRGe bool CodeGeneratorARM::generatePrologue() { MOZ_ASSERT(masm.framePushed() == 0); MOZ_ASSERT(!gen->compilingAsmJS()); #ifdef JS_USE_LINK_REGISTER masm.pushReturnAddress(); #endif + + // If profiling, save the current frame pointer to a per-thread global field. + if (isProfilerInstrumentationEnabled()) + masm.profilerEnterFrame(StackPointer, CallTempReg0); + // Note that this automatically sets MacroAssembler::framePushed(). masm.reserveStack(frameSize()); masm.checkStackAlignment(); emitTracelogIonStart(); return true; } @@ -61,16 +66,22 @@ CodeGeneratorARM::generateEpilogue() { MOZ_ASSERT(!gen->compilingAsmJS()); masm.bind(&returnLabel_); emitTracelogIonStop(); masm.freeStack(frameSize()); MOZ_ASSERT(masm.framePushed() == 0); + + // If profiling, reset the per-thread global lastJitFrame to point to + // the previous frame. + if (isProfilerInstrumentationEnabled()) + masm.profilerExitFrame(); + masm.pop(pc); masm.flushBuffer(); return true; } void CodeGeneratorARM::emitBranch(Assembler::Condition cond, MBasicBlock *mirTrue, MBasicBlock *mirFalse) {
--- a/js/src/jit/arm/MacroAssembler-arm.cpp +++ b/js/src/jit/arm/MacroAssembler-arm.cpp @@ -4289,16 +4289,28 @@ MacroAssemblerARMCompat::handleFailureWi // Only used in debug mode. Return BaselineFrame->returnValue() to the // caller. bind(&return_); ma_ldr(Operand(sp, offsetof(ResumeFromException, framePointer)), r11); ma_ldr(Operand(sp, offsetof(ResumeFromException, stackPointer)), sp); loadValue(Address(r11, BaselineFrame::reverseOffsetOfReturnValue()), JSReturnOperand); ma_mov(r11, sp); pop(r11); + + // If profiling is enabled, then update the lastProfilingFrame to refer to caller + // frame before returning. + { + Label skipProfilingInstrumentation; + // Test if profiler enabled. + AbsoluteAddress addressOfEnabled(GetJitContext()->runtime->spsProfiler().addressOfEnabled()); + branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &skipProfilingInstrumentation); + profilerExitFrame(); + bind(&skipProfilingInstrumentation); + } + ret(); // If we are bailing out to baseline to handle an exception, jump to the // bailout tail stub. bind(&bailout); ma_ldr(Operand(sp, offsetof(ResumeFromException, bailoutInfo)), r2); ma_mov(Imm32(BAILOUT_RETURN_OK), r0); ma_ldr(Operand(sp, offsetof(ResumeFromException, target)), r1);
--- a/js/src/jit/arm/Trampoline-arm.cpp +++ b/js/src/jit/arm/Trampoline-arm.cpp @@ -278,16 +278,29 @@ JitRuntime::generateEnterJIT(JSContext * MOZ_ASSERT(jitcode != ReturnReg); Label error; masm.addPtr(Imm32(ExitFrameLayout::SizeWithFooter()), sp); masm.addPtr(Imm32(BaselineFrame::Size()), framePtr); masm.branchIfFalseBool(ReturnReg, &error); + // If OSR-ing, then emit instrumentation for setting lastProfilerFrame + // if profiler instrumentation is enabled. + { + Label skipProfilingInstrumentation; + Register realFramePtr = numStackValues; + AbsoluteAddress addressOfEnabled(cx->runtime()->spsProfiler.addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), + &skipProfilingInstrumentation); + masm.ma_add(framePtr, Imm32(sizeof(void*)), realFramePtr); + masm.profilerEnterFrame(realFramePtr, scratch); + masm.bind(&skipProfilingInstrumentation); + } + masm.jump(jitcode); // OOM: Load error value, discard return address and previous frame // pointer and return. masm.bind(&error); masm.mov(framePtr, sp); masm.addPtr(Imm32(2 * sizeof(uintptr_t)), sp); masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand); @@ -944,16 +957,27 @@ JitRuntime::generateDebugTrapHandler(JSC masm.branchTest32(Assembler::NonZero, ReturnReg, ReturnReg, &forcedReturn); masm.mov(lr, pc); masm.bind(&forcedReturn); masm.loadValue(Address(r11, BaselineFrame::reverseOffsetOfReturnValue()), JSReturnOperand); masm.mov(r11, sp); masm.pop(r11); + + // Before returning, if profiling is turned on, make sure that lastProfilingFrame + // is set to the correct caller frame. + { + Label skipProfilingInstrumentation; + AbsoluteAddress addressOfEnabled(cx->runtime()->spsProfiler.addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &skipProfilingInstrumentation); + masm.profilerExitFrame(); + masm.bind(&skipProfilingInstrumentation); + } + masm.ret(); Linker linker(masm); AutoFlushICache afc("DebugTrapHandler"); JitCode *codeDbg = linker.newCode<NoGC>(cx, OTHER_CODE); #ifdef JS_ION_PERF writePerfSpewerJitCodeProfile(codeDbg, "DebugTrapHandler");
--- a/js/src/jit/mips/CodeGenerator-mips.cpp +++ b/js/src/jit/mips/CodeGenerator-mips.cpp @@ -40,16 +40,20 @@ CodeGeneratorMIPS::CodeGeneratorMIPS(MIR } bool CodeGeneratorMIPS::generatePrologue() { MOZ_ASSERT(masm.framePushed() == 0); MOZ_ASSERT(!gen->compilingAsmJS()); + // If profiling, save the current frame pointer to a per-thread global field. + if (isProfilerInstrumentationEnabled()) + masm.profilerEnterFrame(StackPointer, CallTempReg0); + // Note that this automatically sets MacroAssembler::framePushed(). masm.reserveStack(frameSize()); masm.checkStackAlignment(); emitTracelogIonStart(); return true; } @@ -59,16 +63,22 @@ CodeGeneratorMIPS::generateEpilogue() { MOZ_ASSERT(!gen->compilingAsmJS()); masm.bind(&returnLabel_); emitTracelogIonStop(); masm.freeStack(frameSize()); MOZ_ASSERT(masm.framePushed() == 0); + + // If profiling, reset the per-thread global lastJitFrame to point to + // the previous frame. + if (isProfilerInstrumentationEnabled()) + masm.profilerExitFrame(); + masm.ret(); return true; } void CodeGeneratorMIPS::branchToBlock(Assembler::FloatFormat fmt, FloatRegister lhs, FloatRegister rhs, MBasicBlock *mir, Assembler::DoubleCondition cond) {
--- a/js/src/jit/mips/MacroAssembler-mips.cpp +++ b/js/src/jit/mips/MacroAssembler-mips.cpp @@ -3599,16 +3599,28 @@ MacroAssemblerMIPSCompat::handleFailureW // caller. bind(&return_); ma_lw(BaselineFrameReg, Address(StackPointer, offsetof(ResumeFromException, framePointer))); ma_lw(StackPointer, Address(StackPointer, offsetof(ResumeFromException, stackPointer))); loadValue(Address(BaselineFrameReg, BaselineFrame::reverseOffsetOfReturnValue()), JSReturnOperand); ma_move(StackPointer, BaselineFrameReg); pop(BaselineFrameReg); + + // If profiling is enabled, then update the lastProfilingFrame to refer to caller + // frame before returning. + { + Label skipProfilingInstrumentation; + // Test if profiler enabled. + AbsoluteAddress addressOfEnabled(GetJitContext()->runtime->spsProfiler().addressOfEnabled()); + branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &skipProfilingInstrumentation); + profilerExitFrame(); + bind(&skipProfilingInstrumentation); + } + ret(); // If we are bailing out to baseline to handle an exception, jump to // the bailout tail stub. bind(&bailout); ma_lw(a2, Address(sp, offsetof(ResumeFromException, bailoutInfo))); ma_li(ReturnReg, Imm32(BAILOUT_RETURN_OK)); ma_lw(a1, Address(sp, offsetof(ResumeFromException, target)));
--- a/js/src/jit/mips/Trampoline-mips.cpp +++ b/js/src/jit/mips/Trampoline-mips.cpp @@ -204,20 +204,23 @@ JitRuntime::generateEnterJIT(JSContext * Label notOsr; masm.ma_b(OsrFrameReg, OsrFrameReg, ¬Osr, Assembler::Zero, ShortJump); Register scratch = regs.takeAny(); Register numStackValues = regs.takeAny(); masm.load32(slotNumStackValues, numStackValues); - // Push return address, previous frame pointer. - masm.subPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer); + // Push return address. + masm.subPtr(Imm32(sizeof(uintptr_t)), StackPointer); masm.ma_li(scratch, returnLabel.dest()); - masm.storePtr(scratch, Address(StackPointer, sizeof(uintptr_t))); + masm.storePtr(scratch, Address(StackPointer, 0)); + + // Push previous frame pointer. + masm.subPtr(Imm32(sizeof(uintptr_t)), StackPointer); masm.storePtr(BaselineFrameReg, Address(StackPointer, 0)); // Reserve frame. Register framePtr = BaselineFrameReg; masm.subPtr(Imm32(BaselineFrame::Size()), StackPointer); masm.movePtr(StackPointer, framePtr); // Reserve space for locals and stack values. @@ -256,16 +259,29 @@ JitRuntime::generateEnterJIT(JSContext * masm.loadPtr(Address(StackPointer, sizeof(uintptr_t)), framePtr); masm.freeStack(2 * sizeof(uintptr_t)); Label error; masm.freeStack(ExitFrameLayout::SizeWithFooter()); masm.addPtr(Imm32(BaselineFrame::Size()), framePtr); masm.branchIfFalseBool(ReturnReg, &error); + // If OSR-ing, then emit instrumentation for setting lastProfilerFrame + // if profiler instrumentation is enabled. + { + Label skipProfilingInstrumentation; + Register realFramePtr = numStackValues; + AbsoluteAddress addressOfEnabled(cx->runtime()->spsProfiler.addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), + &skipProfilingInstrumentation); + masm.ma_add(realFramePtr, StackPointer, Imm32(sizeof(void*))); + masm.profilerEnterFrame(realFramePtr, scratch); + masm.bind(&skipProfilingInstrumentation); + } + masm.jump(jitcode); // OOM: load error value, discard return address and previous frame // pointer and return. masm.bind(&error); masm.movePtr(framePtr, StackPointer); masm.addPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer); masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand); @@ -947,16 +963,27 @@ JitRuntime::generateDebugTrapHandler(JSC // ra was restored by EmitLeaveStubFrame masm.branch(ra); masm.bind(&forcedReturn); masm.loadValue(Address(s5, BaselineFrame::reverseOffsetOfReturnValue()), JSReturnOperand); masm.movePtr(s5, StackPointer); masm.pop(s5); + + // Before returning, if profiling is turned on, make sure that lastProfilingFrame + // is set to the correct caller frame. + { + Label skipProfilingInstrumentation; + AbsoluteAddress addressOfEnabled(cx->runtime()->spsProfiler.addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &skipProfilingInstrumentation); + masm.profilerExitFrame(); + masm.bind(&skipProfilingInstrumentation); + } + masm.ret(); Linker linker(masm); AutoFlushICache afc("DebugTrapHandler"); JitCode *codeDbg = linker.newCode<NoGC>(cx, OTHER_CODE); #ifdef JS_ION_PERF writePerfSpewerJitCodeProfile(codeDbg, "DebugTrapHandler");
--- a/js/src/jit/shared/BaselineCompiler-shared.cpp +++ b/js/src/jit/shared/BaselineCompiler-shared.cpp @@ -26,16 +26,18 @@ BaselineCompilerShared::BaselineCompiler frame(script, masm), stubSpace_(), icEntries_(), pcMappingEntries_(), icLoadLabels_(), pushedBeforeCall_(0), inCall_(false), spsPushToggleOffset_(), + profilerEnterFrameToggleOffset_(), + profilerExitFrameToggleOffset_(), traceLoggerEnterToggleOffset_(), traceLoggerExitToggleOffset_(), traceLoggerScriptTextIdOffset_() { } bool BaselineCompilerShared::callVM(const VMFunction &fun, CallVMPhase phase) {
--- a/js/src/jit/shared/BaselineCompiler-shared.h +++ b/js/src/jit/shared/BaselineCompiler-shared.h @@ -63,16 +63,18 @@ class BaselineCompilerShared CodeOffsetLabel label; }; js::Vector<ICLoadLabel, 16, SystemAllocPolicy> icLoadLabels_; uint32_t pushedBeforeCall_; mozilla::DebugOnly<bool> inCall_; CodeOffsetLabel spsPushToggleOffset_; + CodeOffsetLabel profilerEnterFrameToggleOffset_; + CodeOffsetLabel profilerExitFrameToggleOffset_; CodeOffsetLabel traceLoggerEnterToggleOffset_; CodeOffsetLabel traceLoggerExitToggleOffset_; CodeOffsetLabel traceLoggerScriptTextIdOffset_; BaselineCompilerShared(JSContext *cx, TempAllocator &alloc, JSScript *script); ICEntry *allocateICEntry(ICStub *stub, ICEntry::Kind kind) { if (!stub)
--- a/js/src/jit/shared/CodeGenerator-shared.cpp +++ b/js/src/jit/shared/CodeGenerator-shared.cpp @@ -153,17 +153,17 @@ CodeGeneratorShared::addOutOfLineCode(Ou MOZ_ASSERT_IF(!gen->compilingAsmJS(), code->script()->containsPC(code->pc())); masm.propagateOOM(outOfLineCode_.append(code)); } bool CodeGeneratorShared::addNativeToBytecodeEntry(const BytecodeSite *site) { // Skip the table entirely if profiling is not enabled. - if (!isNativeToBytecodeMapEnabled()) + if (!isProfilerInstrumentationEnabled()) return true; MOZ_ASSERT(site); MOZ_ASSERT(site->tree()); MOZ_ASSERT(site->pc()); InlineScriptTree *tree = site->tree(); jsbytecode *pc = site->pc();
--- a/js/src/jit/shared/CodeGenerator-shared.h +++ b/js/src/jit/shared/CodeGenerator-shared.h @@ -113,18 +113,18 @@ class CodeGeneratorShared : public LElem JSScript **nativeToBytecodeScriptList_; uint32_t nativeToBytecodeScriptListLength_; // When profiling is enabled, this is the instrumentation manager which // maintains state of what script is currently being generated (for inline // scripts) and when instrumentation needs to be emitted or skipped. IonInstrumentation sps_; - bool isNativeToBytecodeMapEnabled() { - return gen->isNativeToBytecodeMapEnabled(); + bool isProfilerInstrumentationEnabled() { + return gen->isProfilerInstrumentationEnabled(); } protected: // The offset of the first instruction of the OSR entry block from the // beginning of the code buffer. size_t osrEntryOffset_; TempAllocator &alloc() const {
--- a/js/src/jit/shared/CodeGenerator-x86-shared.cpp +++ b/js/src/jit/shared/CodeGenerator-x86-shared.cpp @@ -39,16 +39,20 @@ CodeGeneratorX86Shared::CodeGeneratorX86 } bool CodeGeneratorX86Shared::generatePrologue() { MOZ_ASSERT(masm.framePushed() == 0); MOZ_ASSERT(!gen->compilingAsmJS()); + // If profiling, save the current frame pointer to a per-thread global field. + if (isProfilerInstrumentationEnabled()) + masm.profilerEnterFrame(StackPointer, CallTempReg0); + // Note that this automatically sets MacroAssembler::framePushed(). masm.reserveStack(frameSize()); emitTracelogIonStart(); return true; } @@ -60,16 +64,21 @@ CodeGeneratorX86Shared::generateEpilogue masm.bind(&returnLabel_); emitTracelogIonStop(); // Pop the stack we allocated at the start of the function. masm.freeStack(frameSize()); MOZ_ASSERT(masm.framePushed() == 0); + // If profiling, reset the per-thread global lastJitFrame to point to + // the previous frame. + if (isProfilerInstrumentationEnabled()) + masm.profilerExitFrame(); + masm.ret(); return true; } void OutOfLineBailout::accept(CodeGeneratorX86Shared *codegen) { codegen->visitOutOfLineBailout(this);
--- a/js/src/jit/x64/MacroAssembler-x64.cpp +++ b/js/src/jit/x64/MacroAssembler-x64.cpp @@ -436,16 +436,27 @@ MacroAssemblerX64::handleFailureWithHand // Only used in debug mode. Return BaselineFrame->returnValue() to the caller. bind(&return_); loadPtr(Address(rsp, offsetof(ResumeFromException, framePointer)), rbp); loadPtr(Address(rsp, offsetof(ResumeFromException, stackPointer)), rsp); loadValue(Address(rbp, BaselineFrame::reverseOffsetOfReturnValue()), JSReturnOperand); movq(rbp, rsp); pop(rbp); + + // If profiling is enabled, then update the lastProfilingFrame to refer to caller + // frame before returning. + { + Label skipProfilingInstrumentation; + AbsoluteAddress addressOfEnabled(GetJitContext()->runtime->spsProfiler().addressOfEnabled()); + branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &skipProfilingInstrumentation); + profilerExitFrame(); + bind(&skipProfilingInstrumentation); + } + ret(); // If we are bailing out to baseline to handle an exception, jump to // the bailout tail stub. bind(&bailout); loadPtr(Address(esp, offsetof(ResumeFromException, bailoutInfo)), r9); mov(ImmWord(BAILOUT_RETURN_OK), rax); jmp(Operand(rsp, offsetof(ResumeFromException, target)));
--- a/js/src/jit/x64/Trampoline-x64.cpp +++ b/js/src/jit/x64/Trampoline-x64.cpp @@ -160,19 +160,21 @@ JitRuntime::generateEnterJIT(JSContext * Register scratch = regs.takeAny(); Label notOsr; masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, ¬Osr); Register numStackValues = regs.takeAny(); masm.movq(numStackValuesAddr, numStackValues); - // Push return address, previous frame pointer. + // Push return address masm.mov(returnLabel.dest(), scratch); masm.push(scratch); + + // Push previous frame pointer. masm.push(rbp); // Reserve frame. Register framePtr = rbp; masm.subPtr(Imm32(BaselineFrame::Size()), rsp); masm.mov(rsp, framePtr); #ifdef XP_WIN @@ -226,16 +228,29 @@ JitRuntime::generateEnterJIT(JSContext * MOZ_ASSERT(reg_code != ReturnReg); Label error; masm.addPtr(Imm32(ExitFrameLayout::SizeWithFooter()), rsp); masm.addPtr(Imm32(BaselineFrame::Size()), framePtr); masm.branchIfFalseBool(ReturnReg, &error); + // If OSR-ing, then emit instrumentation for setting lastProfilerFrame + // if profiler instrumentation is enabled. + { + Label skipProfilingInstrumentation; + Register realFramePtr = numStackValues; + AbsoluteAddress addressOfEnabled(cx->runtime()->spsProfiler.addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), + &skipProfilingInstrumentation); + masm.lea(Operand(framePtr, sizeof(void*)), realFramePtr); + masm.profilerEnterFrame(realFramePtr, scratch); + masm.bind(&skipProfilingInstrumentation); + } + masm.jump(reg_code); // OOM: load error value, discard return address and previous frame // pointer and return. masm.bind(&error); masm.mov(framePtr, rsp); masm.addPtr(Imm32(2 * sizeof(uintptr_t)), rsp); masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand); @@ -770,16 +785,27 @@ JitRuntime::generateDebugTrapHandler(JSC masm.branchTest32(Assembler::NonZero, ReturnReg, ReturnReg, &forcedReturn); masm.ret(); masm.bind(&forcedReturn); masm.loadValue(Address(ebp, BaselineFrame::reverseOffsetOfReturnValue()), JSReturnOperand); masm.mov(rbp, rsp); masm.pop(rbp); + + // Before returning, if profiling is turned on, make sure that lastProfilingFrame + // is set to the correct caller frame. + { + Label skipProfilingInstrumentation; + AbsoluteAddress addressOfEnabled(cx->runtime()->spsProfiler.addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &skipProfilingInstrumentation); + masm.profilerExitFrame(); + masm.bind(&skipProfilingInstrumentation); + } + masm.ret(); Linker linker(masm); JitCode *codeDbg = linker.newCode<NoGC>(cx, OTHER_CODE); #ifdef JS_ION_PERF writePerfSpewerJitCodeProfile(codeDbg, "DebugTrapHandler"); #endif
--- a/js/src/jit/x86/MacroAssembler-x86.cpp +++ b/js/src/jit/x86/MacroAssembler-x86.cpp @@ -416,16 +416,28 @@ MacroAssemblerX86::handleFailureWithHand // Only used in debug mode. Return BaselineFrame->returnValue() to the caller. bind(&return_); loadPtr(Address(esp, offsetof(ResumeFromException, framePointer)), ebp); loadPtr(Address(esp, offsetof(ResumeFromException, stackPointer)), esp); loadValue(Address(ebp, BaselineFrame::reverseOffsetOfReturnValue()), JSReturnOperand); movl(ebp, esp); pop(ebp); + + // If profiling is enabled, then update the lastProfilingFrame to refer to caller + // frame before returning. + { + Label skipProfilingInstrumentation; + // Test if profiler enabled. + AbsoluteAddress addressOfEnabled(GetJitContext()->runtime->spsProfiler().addressOfEnabled()); + branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &skipProfilingInstrumentation); + profilerExitFrame(); + bind(&skipProfilingInstrumentation); + } + ret(); // If we are bailing out to baseline to handle an exception, jump to // the bailout tail stub. bind(&bailout); loadPtr(Address(esp, offsetof(ResumeFromException, bailoutInfo)), ecx); movl(Imm32(BAILOUT_RETURN_OK), eax); jmp(Operand(esp, offsetof(ResumeFromException, target)));
--- a/js/src/jit/x86/Trampoline-x86.cpp +++ b/js/src/jit/x86/Trampoline-x86.cpp @@ -154,19 +154,21 @@ JitRuntime::generateEnterJIT(JSContext * masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, ¬Osr); Register numStackValues = regs.takeAny(); masm.loadPtr(Address(ebp, ARG_STACKVALUES), numStackValues); Register jitcode = regs.takeAny(); masm.loadPtr(Address(ebp, ARG_JITCODE), jitcode); - // Push return address, previous frame pointer. + // Push return address. masm.mov(returnLabel.dest(), scratch); masm.push(scratch); + + // Push previous frame pointer. masm.push(ebp); // Reserve frame. Register framePtr = ebp; masm.subPtr(Imm32(BaselineFrame::Size()), esp); masm.mov(esp, framePtr); #ifdef XP_WIN @@ -217,16 +219,29 @@ JitRuntime::generateEnterJIT(JSContext * MOZ_ASSERT(jitcode != ReturnReg); Label error; masm.addPtr(Imm32(ExitFrameLayout::SizeWithFooter()), esp); masm.addPtr(Imm32(BaselineFrame::Size()), framePtr); masm.branchIfFalseBool(ReturnReg, &error); + // If OSR-ing, then emit instrumentation for setting lastProfilerFrame + // if profiler instrumentation is enabled. + { + Label skipProfilingInstrumentation; + Register realFramePtr = numStackValues; + AbsoluteAddress addressOfEnabled(cx->runtime()->spsProfiler.addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), + &skipProfilingInstrumentation); + masm.lea(Operand(framePtr, sizeof(void*)), realFramePtr); + masm.profilerEnterFrame(realFramePtr, scratch); + masm.bind(&skipProfilingInstrumentation); + } + masm.jump(jitcode); // OOM: load error value, discard return address and previous frame // pointer and return. masm.bind(&error); masm.mov(framePtr, esp); masm.addPtr(Imm32(2 * sizeof(uintptr_t)), esp); masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand); @@ -808,16 +823,27 @@ JitRuntime::generateDebugTrapHandler(JSC masm.branchTest32(Assembler::NonZero, ReturnReg, ReturnReg, &forcedReturn); masm.ret(); masm.bind(&forcedReturn); masm.loadValue(Address(ebp, BaselineFrame::reverseOffsetOfReturnValue()), JSReturnOperand); masm.mov(ebp, esp); masm.pop(ebp); + + // Before returning, if profiling is turned on, make sure that lastProfilingFrame + // is set to the correct caller frame. + { + Label skipProfilingInstrumentation; + AbsoluteAddress addressOfEnabled(cx->runtime()->spsProfiler.addressOfEnabled()); + masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &skipProfilingInstrumentation); + masm.profilerExitFrame(); + masm.bind(&skipProfilingInstrumentation); + } + masm.ret(); Linker linker(masm); JitCode *codeDbg = linker.newCode<NoGC>(cx, OTHER_CODE); #ifdef JS_ION_PERF writePerfSpewerJitCodeProfile(codeDbg, "DebugTrapHandler"); #endif
--- a/js/src/vm/Runtime.h +++ b/js/src/vm/Runtime.h @@ -589,16 +589,19 @@ class PerThreadData : public PerThreadDa } static unsigned offsetOfActivation() { return offsetof(PerThreadData, activation_); } js::Activation *profilingActivation() const { return profilingActivation_; } + void *addressOfProfilingActivation() { + return (void*) &profilingActivation_; + } js::AsmJSActivation *asmJSActivationStack() const { return asmJSActivationStack_; } static js::AsmJSActivation *innermostAsmJSActivation() { PerThreadData *ptd = TlsPerThreadData.get(); return ptd ? ptd->asmJSActivationStack_ : nullptr; }
--- a/js/src/vm/SPSProfiler.cpp +++ b/js/src/vm/SPSProfiler.cpp @@ -7,17 +7,19 @@ #include "vm/SPSProfiler.h" #include "mozilla/DebugOnly.h" #include "jsnum.h" #include "jsprf.h" #include "jsscript.h" +#include "jit/BaselineFrame.h" #include "jit/BaselineJIT.h" +#include "jit/JitFrames.h" #include "vm/StringBuffer.h" using namespace js; using mozilla::DebugOnly; SPSProfiler::SPSProfiler(JSRuntime *rt) : rt(rt), @@ -87,16 +89,24 @@ SPSProfiler::enable(bool enabled) enabled_ = enabled; /* Toggle SPS-related jumps on baseline jitcode. * The call to |ReleaseAllJITCode| above will release most baseline jitcode, but not * jitcode for scripts with active frames on the stack. These scripts need to have * their profiler state toggled so they behave properly. */ jit::ToggleBaselineSPS(rt, enabled); + + /* Update lastProfilingFrame to point to the top-most JS jit-frame currently on + * stack. + */ + if (rt->mainThread.jitActivation) { + void *lastProfilingFrame = GetTopProfilingJitFrame(rt->mainThread.jitTop); + rt->mainThread.jitActivation->setLastProfilingFrame(lastProfilingFrame); + } } /* Lookup the string for the function/script, creating one if necessary */ const char* SPSProfiler::profileString(JSScript *script, JSFunction *maybeFun) { AutoSPSLock lock(lock_); MOZ_ASSERT(strings.initialized()); @@ -397,8 +407,41 @@ AutoSuppressProfilerSampling::AutoSuppre rt_->disableProfilerSampling(); } AutoSuppressProfilerSampling::~AutoSuppressProfilerSampling() { if (previouslyEnabled_) rt_->enableProfilerSampling(); } + +void * +js::GetTopProfilingJitFrame(uint8_t *exitFramePtr) +{ + // For null exitFrame, there is no previous exit frame, just return. + if (!exitFramePtr) + return nullptr; + + jit::ExitFrameLayout *exitFrame = (jit::ExitFrameLayout *) exitFramePtr; + size_t prevSize = exitFrame->prevFrameLocalSize(); + jit::FrameType prevType = exitFrame->prevType(); + + uint8_t *prev = exitFramePtr + (jit::ExitFrameLayout::Size() + prevSize); + + // previous frame type must be one of IonJS, BaselineJS, or BaselineStub, + // or unwound variants thereof. + switch (prevType) { + case jit::JitFrame_IonJS: + case jit::JitFrame_Unwound_IonJS: + case jit::JitFrame_BaselineJS: + return prev; + + case jit::JitFrame_BaselineStub: + case jit::JitFrame_Unwound_BaselineStub: { + void *framePtr = ((jit::BaselineStubFrameLayout *) prev)->reverseSavedFramePtr(); + return ((uint8_t *) framePtr) + jit::BaselineFrame::FramePointerOffset; + } + + default: + MOZ_CRASH("unknown callee token type"); + return nullptr; + } +}
--- a/js/src/vm/SPSProfiler.h +++ b/js/src/vm/SPSProfiler.h @@ -508,11 +508,15 @@ class SPSInstrumentation MOZ_ASSERT(frame->left == 0); MOZ_ASSERT(frame->script); if (!inlinedFunction) masm.spsPopFrame(profiler_, scratch); } } }; + +/* Get a pointer to the top-most profiling frame, given the exit frame pointer. */ +void *GetTopProfilingJitFrame(uint8_t *exitFramePtr); + } /* namespace js */ #endif /* vm_SPSProfiler_h */