Bug 1382650 part 7 - Use a separate Ion optimization level for very hot code. r=nbp a=pascalc
☠☠ backed out by a330a9835751 ☠ ☠
authorNarcis Beleuzu <nbeleuzu@mozilla.com>
Sun, 07 Apr 2019 16:42:01 +0300
changeset 526000 dfde2a76520d0f0539a4ee1b377c34f5d410d9a3
parent 525999 e099b8601dc77a26fbaaf6cfef7254cb4a9d5247
child 526001 353c9e1559f09f75eafcb6f7897b21a75777e64a
push id2032
push userffxbld-merge
push dateMon, 13 May 2019 09:36:57 +0000
treeherdermozilla-release@455c1065dcbe [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersnbp, pascalc
bugs1382650
milestone67.0
Bug 1382650 part 7 - Use a separate Ion optimization level for very hot code. r=nbp a=pascalc Summary: Ion can do aggressive inlining, but inlining a lot of code has a negative effect on compilation time and memory usage. It also means we spend more time in the slower Baseline code while compiling the Ion code off-thread or after an invalidation. To address this, Ion now consists of two tiers: * Normal: the first tier (warm-up threshold of 1,000) only inlines small functions one level deep. This tier also has recompile checks to recompile the script when it becomes very hot. * Full: the second tier (warm-up threshold of 100,000) is only used for very hot code so we can afford inlining a lot more code. This improves Speedometer and GDocs by more than 5%. Reviewers: nbp Reviewed By: nbp Bug #: 1382650 Differential Revision: https://phabricator.services.mozilla.com/D24159
js/src/jit-test/lib/jitopts.js
js/src/jit-test/tests/asm.js/testBug1125561.js
js/src/jit-test/tests/ion/dce-with-rinstructions.js
js/src/jit-test/tests/ion/recover-lambdas.js
js/src/jit/CodeGenerator.cpp
js/src/jit/Ion.cpp
js/src/jit/IonAnalysis.cpp
js/src/jit/IonBuilder.cpp
js/src/jit/IonBuilder.h
js/src/jit/IonOptimizationLevels.cpp
js/src/jit/IonOptimizationLevels.h
js/src/jit/MIR.h
js/src/jit/VMFunctionList-inl.h
js/src/jit/VMFunctions.cpp
js/src/jit/VMFunctions.h
--- a/js/src/jit-test/lib/jitopts.js
+++ b/js/src/jit-test/lib/jitopts.js
@@ -28,38 +28,41 @@ function withJitOptions(opts, fn) {
 // N.B. Ion opts *must come before* baseline opts because there's some kind of
 // "undo eager compilation" logic. If we don't set the baseline warmup-counter
 // *after* the Ion warmup-counter we end up setting the baseline warmup-counter
 // to be the default if we hit the "undo eager compilation" logic.
 var Opts_BaselineEager =
     {
       'ion.enable': 1,
       'ion.warmup.trigger': 100,
+      'ion.full.warmup.trigger': 100,
       'baseline.enable': 1,
       'baseline.warmup.trigger': 0,
       'offthread-compilation.enable': 1
     };
 
 // Checking for offthread compilation being off is often helpful if the test
 // requires a function be Ion compiled. Each individual test will usually
 // finish before the Ion compilation thread has a chance to attach the
 // compiled code.
 var Opts_IonEagerNoOffthreadCompilation =
     {
       'ion.enable': 1,
       'ion.warmup.trigger': 0,
+      'ion.full.warmup.trigger': 0,
       'baseline.enable': 1,
       'baseline.warmup.trigger': 0,
       'offthread-compilation.enable': 0,
     };
 
 var Opts_Ion2NoOffthreadCompilation =
     {
       'ion.enable': 1,
       'ion.warmup.trigger': 2,
+      'ion.full.warmup.trigger': 2,
       'baseline.enable': 1,
       'baseline.warmup.trigger': 1,
       'offthread-compilation.enable': 0
     };
 
 var Opts_NoJits =
     {
       'ion.enable': 0,
--- a/js/src/jit-test/tests/asm.js/testBug1125561.js
+++ b/js/src/jit-test/tests/asm.js/testBug1125561.js
@@ -1,11 +1,11 @@
 load(libdir + "asm.js");
 
-setJitCompilerOption("ion.warmup.trigger", 0);
+setJitCompilerOption("ion.full.warmup.trigger", 0);
 setJitCompilerOption("baseline.warmup.trigger", 0);
 setJitCompilerOption("offthread-compilation.enable", 0);
 
 function ffi1() { assertJitStackInvariants() }
 function ffi2() { return { valueOf() { assertJitStackInvariants() } } }
 
 // FFI with no coercion
 var m = asmCompile('stdlib', 'foreign', `
--- a/js/src/jit-test/tests/ion/dce-with-rinstructions.js
+++ b/js/src/jit-test/tests/ion/dce-with-rinstructions.js
@@ -1,10 +1,11 @@
 setJitCompilerOption("baseline.warmup.trigger", 10);
 setJitCompilerOption("ion.warmup.trigger", 20);
+setJitCompilerOption("ion.full.warmup.trigger", 20);
 var i;
 
 var config = getBuildConfiguration();
 var max = 200;
 
 // Check that we are able to remove the operation inside recover test functions (denoted by "rop..."),
 // when we inline the first version of uceFault, and ensure that the bailout is correct
 // when uceFault is replaced (which cause an invalidation bailout)
--- a/js/src/jit-test/tests/ion/recover-lambdas.js
+++ b/js/src/jit-test/tests/ion/recover-lambdas.js
@@ -1,12 +1,13 @@
 // |jit-test| --ion-osr=off
 
 var max = 40;
 setJitCompilerOption("ion.warmup.trigger", max - 10);
+setJitCompilerOption("ion.full.warmup.trigger", max - 10);
 
 // This function is used to escape "g" which is a non-escaped inner function.
 // As it is not escaped within "f", the lambda for "g" would be computed on the
 // bailout path. Resolving the first ".caller" implies that we have to recover
 // the lambda. Resolving the second ".caller" is needed such as we can build the
 // test case without explicitly escaping "g", which would prevent this
 // optimization.
 
--- a/js/src/jit/CodeGenerator.cpp
+++ b/js/src/jit/CodeGenerator.cpp
@@ -13434,47 +13434,56 @@ void CodeGenerator::visitWasmLoadTls(LWa
     default:
       MOZ_CRASH("MIRType not supported in WasmLoadTls");
   }
 }
 
 void CodeGenerator::visitRecompileCheck(LRecompileCheck* ins) {
   Label done;
   Register tmp = ToRegister(ins->scratch());
-  OutOfLineCode* ool;
-
-  using Fn = bool (*)(JSContext*);
-  if (ins->mir()->forceRecompilation()) {
-    ool = oolCallVM<Fn, IonForcedRecompile>(ins, ArgList(), StoreNothing());
-  } else {
-    ool = oolCallVM<Fn, IonRecompile>(ins, ArgList(), StoreNothing());
+
+  OutOfLineCode* ool = nullptr;
+  if (ins->mir()->checkCounter()) {
+    using Fn = bool (*)(JSContext*);
+    if (ins->mir()->forceInvalidation()) {
+      ool =
+          oolCallVM<Fn, IonForcedInvalidation>(ins, ArgList(), StoreNothing());
+    } else if (ins->mir()->forceRecompilation()) {
+      ool = oolCallVM<Fn, IonForcedRecompile>(ins, ArgList(), StoreNothing());
+    } else {
+      ool = oolCallVM<Fn, IonRecompile>(ins, ArgList(), StoreNothing());
+    }
   }
 
   // Check if warm-up counter is high enough.
   AbsoluteAddress warmUpCount =
       AbsoluteAddress(ins->mir()->script()->addressOfWarmUpCounter());
   if (ins->mir()->increaseWarmUpCounter()) {
     masm.load32(warmUpCount, tmp);
     masm.add32(Imm32(1), tmp);
     masm.store32(tmp, warmUpCount);
-    masm.branch32(Assembler::BelowOrEqual, tmp,
-                  Imm32(ins->mir()->recompileThreshold()), &done);
+    if (ins->mir()->checkCounter()) {
+      masm.branch32(Assembler::BelowOrEqual, tmp,
+                    Imm32(ins->mir()->recompileThreshold()), &done);
+    }
   } else {
     masm.branch32(Assembler::BelowOrEqual, warmUpCount,
                   Imm32(ins->mir()->recompileThreshold()), &done);
   }
 
   // Check if not yet recompiling.
-  CodeOffset label = masm.movWithPatch(ImmWord(uintptr_t(-1)), tmp);
-  masm.propagateOOM(ionScriptLabels_.append(label));
-  masm.branch32(Assembler::Equal,
-                Address(tmp, IonScript::offsetOfRecompiling()), Imm32(0),
-                ool->entry());
-  masm.bind(ool->rejoin());
-  masm.bind(&done);
+  if (ins->mir()->checkCounter()) {
+    CodeOffset label = masm.movWithPatch(ImmWord(uintptr_t(-1)), tmp);
+    masm.propagateOOM(ionScriptLabels_.append(label));
+    masm.branch32(Assembler::Equal,
+                  Address(tmp, IonScript::offsetOfRecompiling()), Imm32(0),
+                  ool->entry());
+    masm.bind(ool->rejoin());
+    masm.bind(&done);
+  }
 }
 
 void CodeGenerator::visitLexicalCheck(LLexicalCheck* ins) {
   ValueOperand inputValue = ToValue(ins, LLexicalCheck::Input);
   Label bail;
   masm.branchTestMagicValue(Assembler::Equal, inputValue,
                             JS_UNINITIALIZED_LEXICAL, &bail);
   bailoutFrom(&bail, ins->snapshot());
--- a/js/src/jit/Ion.cpp
+++ b/js/src/jit/Ion.cpp
@@ -2233,16 +2233,20 @@ static MethodStatus Compile(JSContext* c
     return Method_Skipped;
   }
 
   if (!CanLikelyAllocateMoreExecutableMemory()) {
     script->resetWarmUpCounter();
     return Method_Skipped;
   }
 
+  if (script->baselineScript()->hasPendingIonBuilder()) {
+    LinkIonScript(cx, script);
+  }
+
   if (script->hasIonScript()) {
     IonScript* scriptIon = script->ionScript();
     if (!scriptIon->method()) {
       return Method_CantCompile;
     }
 
     // Don't recompile/overwrite higher optimized code,
     // with a lower optimization level.
@@ -2258,26 +2262,16 @@ static MethodStatus Compile(JSContext* c
 
     if (osrPc) {
       scriptIon->resetOsrPcMismatchCounter();
     }
 
     recompile = true;
   }
 
-  if (script->baselineScript()->hasPendingIonBuilder()) {
-    IonBuilder* buildIon = script->baselineScript()->pendingIonBuilder();
-    if (optimizationLevel <= buildIon->optimizationInfo().level() &&
-        !forceRecompile) {
-      return Method_Compiled;
-    }
-
-    recompile = true;
-  }
-
   AbortReason reason =
       IonCompile(cx, script, osrFrame, osrPc, recompile, optimizationLevel);
   if (reason == AbortReason::Error) {
     MOZ_ASSERT(cx->isExceptionPending());
     return Method_Error;
   }
 
   if (reason == AbortReason::Disable) {
@@ -2574,16 +2568,18 @@ bool jit::IonCompileScriptForBaseline(JS
 MethodStatus jit::Recompile(JSContext* cx, HandleScript script,
                             BaselineFrame* osrFrame, jsbytecode* osrPc,
                             bool force) {
   MOZ_ASSERT(script->hasIonScript());
   if (script->ionScript()->isRecompiling()) {
     return Method_Compiled;
   }
 
+  MOZ_ASSERT(!script->baselineScript()->hasPendingIonBuilder());
+
   MethodStatus status = Compile(cx, script, osrFrame, osrPc, force);
   if (status != Method_Compiled) {
     if (status == Method_CantCompile) {
       ForbidCompilation(cx, script);
     }
     return status;
   }
 
--- a/js/src/jit/IonAnalysis.cpp
+++ b/js/src/jit/IonAnalysis.cpp
@@ -4571,17 +4571,17 @@ bool jit::AnalyzeNewScriptDefiniteProper
     return false;
   }
 
   CompileInfo info(CompileRuntime::get(cx->runtime()), script, fun,
                    /* osrPc = */ nullptr, Analysis_DefiniteProperties,
                    script->needsArgsObj(), inlineScriptTree);
 
   const OptimizationInfo* optimizationInfo =
-      IonOptimizations.get(OptimizationLevel::Normal);
+      IonOptimizations.get(OptimizationLevel::Full);
 
   CompilerConstraintList* constraints = NewCompilerConstraintList(temp);
   if (!constraints) {
     ReportOutOfMemory(cx);
     return false;
   }
 
   BaselineInspector inspector(script);
--- a/js/src/jit/IonBuilder.cpp
+++ b/js/src/jit/IonBuilder.cpp
@@ -480,16 +480,29 @@ IonBuilder::InliningDecision IonBuilder:
   }
 
   // Don't inline functions which don't have baseline scripts.
   if (!inlineScript->hasBaselineScript()) {
     trackOptimizationOutcome(TrackedOutcome::CantInlineNoBaseline);
     return DontInline(inlineScript, "No baseline jitcode");
   }
 
+  // Don't inline functions with a higher optimization level.
+  if (!isHighestOptimizationLevel()) {
+    OptimizationLevel level = optimizationLevel();
+    if (inlineScript->hasIonScript() &&
+        (inlineScript->ionScript()->isRecompiling() ||
+         inlineScript->ionScript()->optimizationLevel() > level)) {
+      return DontInline(inlineScript, "More optimized");
+    }
+    if (IonOptimizations.levelForScript(inlineScript, nullptr) > level) {
+      return DontInline(inlineScript, "Should be more optimized");
+    }
+  }
+
   if (TooManyFormalArguments(target->nargs())) {
     trackOptimizationOutcome(TrackedOutcome::CantInlineTooManyArgs);
     return DontInline(inlineScript, "Too many args");
   }
 
   // We check the number of actual arguments against the maximum number of
   // formal arguments as we do not want to encode all actual arguments in the
   // callerResumePoint.
@@ -772,17 +785,20 @@ AbortReasonOr<Ok> IonBuilder::build() {
 #ifdef JS_STRUCTURED_SPEW
   if (!info().isAnalysis()) {
     JitSpewBaselineICStats(script(), "To-Be-Compiled");
   }
 #endif
 
   MOZ_TRY(init());
 
-  if (script()->hasBaselineScript()) {
+  // The BaselineScript-based inlining heuristics only affect the highest
+  // optimization level. Other levels do almost no inlining and we don't want to
+  // overwrite data from the highest optimization tier.
+  if (script()->hasBaselineScript() && isHighestOptimizationLevel()) {
     script()->baselineScript()->resetMaxInliningDepth();
   }
 
   MBasicBlock* entry;
   MOZ_TRY_VAR(entry, newBlock(info().firstStackSlot(), pc));
   MOZ_TRY(setCurrentAndSpecializePhis(entry));
 
 #ifdef JS_JITSPEW
@@ -792,17 +808,17 @@ AbortReasonOr<Ok> IonBuilder::build() {
             (void*)script(), AnalysisModeString(info().analysisMode()));
   } else {
     JitSpew(JitSpew_IonScripts,
             "%sompiling script %s:%u:%u (%p) (warmup-counter=%" PRIu32
             ", level=%s)",
             (script()->hasIonScript() ? "Rec" : "C"), script()->filename(),
             script()->lineno(), script()->column(), (void*)script(),
             script()->getWarmUpCount(),
-            OptimizationLevelString(optimizationInfo().level()));
+            OptimizationLevelString(optimizationLevel()));
   }
 #endif
 
   MOZ_TRY(initParameters());
   initLocals();
 
   // Initialize something for the env chain. We can bail out before the
   // start instruction, but the snapshot is encoded *at* the start
@@ -905,17 +921,17 @@ AbortReasonOr<Ok> IonBuilder::build() {
 
   auto clearLastPriorResumePoint = mozilla::MakeScopeExit([&] {
     // Discard unreferenced & pre-allocated resume points.
     replaceMaybeFallbackFunctionGetter(nullptr);
   });
 
   MOZ_TRY(traverseBytecode());
 
-  if (script_->hasBaselineScript() &&
+  if (isHighestOptimizationLevel() && script_->hasBaselineScript() &&
       inlinedBytecodeLength_ >
           script_->baselineScript()->inlinedBytecodeLength()) {
     script_->baselineScript()->setInlinedBytecodeLength(inlinedBytecodeLength_);
   }
 
   MOZ_TRY(maybeAddOsrTypeBarriers());
   MOZ_TRY(processIterators());
 
@@ -4364,17 +4380,20 @@ IonBuilder::InliningDecision IonBuilder:
 
   BaselineScript* outerBaseline =
       outermostBuilder()->script()->baselineScript();
   if (inliningDepth_ >= maxInlineDepth) {
     // We hit the depth limit and won't inline this function. Give the
     // outermost script a max inlining depth of 0, so that it won't be
     // inlined in other scripts. This heuristic is currently only used
     // when we're inlining scripts with loops, see the comment below.
-    outerBaseline->setMaxInliningDepth(0);
+    // These heuristics only apply to the highest optimization level.
+    if (isHighestOptimizationLevel()) {
+      outerBaseline->setMaxInliningDepth(0);
+    }
 
     trackOptimizationOutcome(TrackedOutcome::CantInlineExceededDepth);
     return DontInline(targetScript, "Vetoed: exceeding allowed inline depth");
   }
 
   // Inlining functions with loops can be complicated. For instance, if we're
   // close to the inlining depth limit and we inline the function f below, we
   // can no longer inline the call to g:
@@ -4387,27 +4406,31 @@ IonBuilder::InliningDecision IonBuilder:
   //
   // If the loop has many iterations, it's more efficient to call f and inline
   // g in f.
   //
   // To avoid this problem, we record a separate max inlining depth for each
   // script, indicating at which depth we won't be able to inline all functions
   // we inlined this time. This solves the issue above, because we will only
   // inline f if it means we can also inline g.
-  if (targetScript->hasLoops() &&
+  //
+  // These heuristics only apply to the highest optimization level: other tiers
+  // do very little inlining and performance is not as much of a concern there.
+  if (isHighestOptimizationLevel() && targetScript->hasLoops() &&
       inliningDepth_ >= targetScript->baselineScript()->maxInliningDepth()) {
     trackOptimizationOutcome(TrackedOutcome::CantInlineExceededDepth);
     return DontInline(targetScript,
                       "Vetoed: exceeding allowed script inline depth");
   }
 
   // Update the max depth at which we can inline the outer script.
   MOZ_ASSERT(maxInlineDepth > inliningDepth_);
   uint32_t scriptInlineDepth = maxInlineDepth - inliningDepth_ - 1;
-  if (scriptInlineDepth < outerBaseline->maxInliningDepth()) {
+  if (scriptInlineDepth < outerBaseline->maxInliningDepth() &&
+      isHighestOptimizationLevel()) {
     outerBaseline->setMaxInliningDepth(scriptInlineDepth);
   }
 
   // End of heuristics, we will inline this function.
 
   outerBuilder->inlinedBytecodeLength_ += targetScript->length();
 
   return InliningDecision_Inline;
@@ -5950,22 +5973,22 @@ AbortReasonOr<Ok> IonBuilder::jsop_call(
       }
       if (!callTargets->append(&target.target->as<JSFunction>())) {
         return abort(AbortReason::Alloc);
       }
     }
   }
 
   if (status == InliningStatus_WarmUpCountTooLow && callTargets &&
-      callTargets->length() == 1) {
+      callTargets->length() == 1 && isHighestOptimizationLevel()) {
     JSFunction* target = callTargets.ref()[0];
     MRecompileCheck* check =
         MRecompileCheck::New(alloc(), target->nonLazyScript(),
                              optimizationInfo().inliningRecompileThreshold(),
-                             MRecompileCheck::RecompileCheck_Inlining);
+                             MRecompileCheck::RecompileCheckType::Inlining);
     current->add(check);
   }
 
   return makeCall(callTargets, callInfo);
 }
 
 AbortReasonOr<bool> IonBuilder::testShouldDOMCall(TypeSet* inTypes,
                                                   JSFunction* func,
@@ -7626,37 +7649,43 @@ static bool ObjectHasExtraOwnProperty(Co
   }
 
   // Resolve hooks can install new properties on objects on demand.
   JSObject* singleton = object->isSingleton() ? object->singleton() : nullptr;
   return ClassMayResolveId(realm->runtime()->names(), clasp, id, singleton);
 }
 
 void IonBuilder::insertRecompileCheck() {
+  MOZ_ASSERT(pc == script()->code() || *pc == JSOP_LOOPENTRY);
+
   // No need for recompile checks if this is the highest optimization level.
-  OptimizationLevel curLevel = optimizationInfo().level();
+  OptimizationLevel curLevel = optimizationLevel();
   if (IonOptimizations.isLastLevel(curLevel)) {
     return;
   }
 
-  // Add recompile check.
-
-  // Get the topmost builder. The topmost script will get recompiled when
-  // warm-up counter is high enough to justify a higher optimization level.
-  IonBuilder* topBuilder = outermostBuilder();
+  // Add recompile check. See MRecompileCheck::RecompileCheckType for how this
+  // works.
+
+  MRecompileCheck::RecompileCheckType type;
+  if (*pc == JSOP_LOOPENTRY) {
+    type = MRecompileCheck::RecompileCheckType::OptimizationLevelOSR;
+  } else if (this != outermostBuilder()) {
+    type = MRecompileCheck::RecompileCheckType::OptimizationLevelInlined;
+  } else {
+    type = MRecompileCheck::RecompileCheckType::OptimizationLevel;
+  }
 
   // Add recompile check to recompile when the warm-up count reaches the
   // threshold of the next optimization level.
   OptimizationLevel nextLevel = IonOptimizations.nextLevel(curLevel);
   const OptimizationInfo* info = IonOptimizations.get(nextLevel);
-  uint32_t warmUpThreshold =
-      info->compilerWarmUpThreshold(topBuilder->script());
+  uint32_t warmUpThreshold = info->recompileWarmUpThreshold(script(), pc);
   MRecompileCheck* check =
-      MRecompileCheck::New(alloc(), topBuilder->script(), warmUpThreshold,
-                           MRecompileCheck::RecompileCheck_OptimizationLevel);
+      MRecompileCheck::New(alloc(), script(), warmUpThreshold, type);
   current->add(check);
 }
 
 JSObject* IonBuilder::testSingletonProperty(JSObject* obj, jsid id) {
   // We would like to completely no-op property/global accesses which can
   // produce only a particular JSObject. When indicating the access result is
   // definitely an object, type inference does not account for the
   // possibility that the property is entirely missing from the input object
--- a/js/src/jit/IonBuilder.h
+++ b/js/src/jit/IonBuilder.h
@@ -699,16 +699,23 @@ class IonBuilder : public MIRGenerator,
   // Oracles.
   InliningDecision canInlineTarget(JSFunction* target, CallInfo& callInfo);
   InliningDecision makeInliningDecision(JSObject* target, CallInfo& callInfo);
   AbortReasonOr<Ok> selectInliningTargets(const InliningTargets& targets,
                                           CallInfo& callInfo,
                                           BoolVector& choiceSet,
                                           uint32_t* numInlineable);
 
+  OptimizationLevel optimizationLevel() const {
+    return optimizationInfo().level();
+  }
+  bool isHighestOptimizationLevel() const {
+    return IonOptimizations.isLastLevel(optimizationLevel());
+  }
+
   // Native inlining helpers.
   // The typeset for the return value of our function.  These are
   // the types it's been observed returning in the past.
   TemporaryTypeSet* getInlineReturnTypeSet();
   // The known MIR type of getInlineReturnTypeSet.
   MIRType getInlineReturnType();
 
   // Array natives.
--- a/js/src/jit/IonOptimizationLevels.cpp
+++ b/js/src/jit/IonOptimizationLevels.cpp
@@ -25,31 +25,43 @@ void OptimizationInfo::initNormalOptimiz
   edgeCaseAnalysis_ = true;
   eliminateRedundantChecks_ = true;
   inlineInterpreted_ = true;
   inlineNative_ = true;
   licm_ = true;
   gvn_ = true;
   rangeAnalysis_ = true;
   reordering_ = true;
+  scalarReplacement_ = true;
   sincos_ = true;
   sink_ = true;
 
   registerAllocator_ = RegisterAllocator_Backtracking;
 
-  inlineMaxBytecodePerCallSiteMainThread_ = 550;
-  inlineMaxBytecodePerCallSiteHelperThread_ = 1100;
+  inlineMaxBytecodePerCallSiteMainThread_ = 200;
+  inlineMaxBytecodePerCallSiteHelperThread_ = 400;
   inlineMaxCalleeInlinedBytecodeLength_ = 3550;
   inlineMaxTotalBytecodeLength_ = 85000;
   inliningMaxCallerBytecodeLength_ = 1600;
+  maxInlineDepth_ = 0;
+  smallFunctionMaxInlineDepth_ = 1;
+  inliningWarmUpThresholdFactor_ = 0.5;
+  inliningRecompileThresholdFactor_ = 4;
+}
+
+void OptimizationInfo::initFullOptimizationInfo() {
+  initNormalOptimizationInfo();
+
+  level_ = OptimizationLevel::Full;
+
+  inlineMaxBytecodePerCallSiteMainThread_ = 550;
+  inlineMaxBytecodePerCallSiteHelperThread_ = 1100;
   maxInlineDepth_ = 3;
-  scalarReplacement_ = true;
   smallFunctionMaxInlineDepth_ = 10;
   inliningWarmUpThresholdFactor_ = 0.125;
-  inliningRecompileThresholdFactor_ = 4;
 }
 
 void OptimizationInfo::initWasmOptimizationInfo() {
   // The Wasm optimization level
   // Disables some passes that don't work well with wasm.
 
   // Take normal option values for not specified values.
   initNormalOptimizationInfo();
@@ -98,18 +110,39 @@ uint32_t OptimizationInfo::compilerWarmU
   // It's more efficient to enter outer loops, rather than inner loops, via OSR.
   // To accomplish this, we use a slightly higher threshold for inner loops.
   // Note that the loop depth is always > 0 so we will prefer non-OSR over OSR.
   uint32_t loopDepth = LoopEntryDepthHint(pc);
   MOZ_ASSERT(loopDepth > 0);
   return warmUpThreshold + loopDepth * (baseCompilerWarmUpThreshold() / 10);
 }
 
+uint32_t OptimizationInfo::recompileWarmUpThreshold(JSScript* script,
+                                                    jsbytecode* pc) const {
+  MOZ_ASSERT(pc == script->code() || *pc == JSOP_LOOPENTRY);
+
+  uint32_t threshold = compilerWarmUpThreshold(script, pc);
+  if (*pc != JSOP_LOOPENTRY || JitOptions.eagerIonCompilation()) {
+    return threshold;
+  }
+
+  // If we're stuck in a long-running loop at a low optimization level, we have
+  // to invalidate to be able to tier up. This is worse than recompiling at
+  // function entry (because in that case we can use the lazy link mechanism and
+  // avoid invalidation completely). Use a very high recompilation threshold for
+  // loop edges so that this only affects very long-running loops.
+
+  uint32_t loopDepth = LoopEntryDepthHint(pc);
+  MOZ_ASSERT(loopDepth > 0);
+  return threshold + loopDepth * (baseCompilerWarmUpThreshold() / 10);
+}
+
 OptimizationLevelInfo::OptimizationLevelInfo() {
   infos_[OptimizationLevel::Normal].initNormalOptimizationInfo();
+  infos_[OptimizationLevel::Full].initFullOptimizationInfo();
   infos_[OptimizationLevel::Wasm].initWasmOptimizationInfo();
 
 #ifdef DEBUG
   OptimizationLevel level = firstLevel();
   while (!isLastLevel(level)) {
     OptimizationLevel next = nextLevel(level);
     MOZ_ASSERT_IF(level != OptimizationLevel::DontCompile, level < next);
     level = next;
@@ -119,28 +152,31 @@ OptimizationLevelInfo::OptimizationLevel
 
 OptimizationLevel OptimizationLevelInfo::nextLevel(
     OptimizationLevel level) const {
   MOZ_ASSERT(!isLastLevel(level));
   switch (level) {
     case OptimizationLevel::DontCompile:
       return OptimizationLevel::Normal;
     case OptimizationLevel::Normal:
+      return OptimizationLevel::Full;
+    case OptimizationLevel::Full:
     case OptimizationLevel::Wasm:
-    case OptimizationLevel::Count:;
+    case OptimizationLevel::Count:
+      break;
   }
   MOZ_CRASH("Unknown optimization level.");
 }
 
 OptimizationLevel OptimizationLevelInfo::firstLevel() const {
   return nextLevel(OptimizationLevel::DontCompile);
 }
 
 bool OptimizationLevelInfo::isLastLevel(OptimizationLevel level) const {
-  return level == OptimizationLevel::Normal;
+  return level == OptimizationLevel::Full;
 }
 
 OptimizationLevel OptimizationLevelInfo::levelForScript(JSScript* script,
                                                         jsbytecode* pc) const {
   OptimizationLevel prev = OptimizationLevel::DontCompile;
 
   while (!isLastLevel(prev)) {
     OptimizationLevel level = nextLevel(prev);
--- a/js/src/jit/IonOptimizationLevels.h
+++ b/js/src/jit/IonOptimizationLevels.h
@@ -12,25 +12,51 @@
 #include "jstypes.h"
 
 #include "jit/JitOptions.h"
 #include "js/TypeDecls.h"
 
 namespace js {
 namespace jit {
 
-enum class OptimizationLevel : uint8_t { Normal, Wasm, Count, DontCompile };
+// [SMDOC] Ion Optimization Levels
+//
+// Ion can do aggressive inlining, but inlining a lot of code will have a
+// negative effect on compilation time and memory usage. It also means we spend
+// more time in the slower Baseline code while compiling the Ion code
+// off-thread or after an invalidation.
+//
+// To address this, Ion consists of two tiers:
+//
+// * Normal: the first tier (warm-up threshold of 1,000) only inlines small
+//           functions one level deep. This tier also has recompile checks to
+//           recompile the script when it becomes very hot.
+//
+// * Full: the second tier (warm-up threshold of 100,000) is only used for very
+//         hot code so we can afford inlining a lot more code.
+//
+// See MRecompileCheck::RecompileCheckType for more info.
+
+enum class OptimizationLevel : uint8_t {
+  Normal,
+  Full,
+  Wasm,
+  Count,
+  DontCompile
+};
 
 #ifdef JS_JITSPEW
 inline const char* OptimizationLevelString(OptimizationLevel level) {
   switch (level) {
     case OptimizationLevel::DontCompile:
       return "Optimization_DontCompile";
     case OptimizationLevel::Normal:
       return "Optimization_Normal";
+    case OptimizationLevel::Full:
+      return "Optimization_Full";
     case OptimizationLevel::Wasm:
       return "Optimization_Wasm";
     case OptimizationLevel::Count:;
   }
   MOZ_CRASH("Invalid OptimizationLevel");
 }
 #endif
 
@@ -119,16 +145,18 @@ class OptimizationInfo {
   // is hot enough to recompile the outerScript to inline that function,
   // as a multiplication of inliningWarmUpThreshold.
   uint32_t inliningRecompileThresholdFactor_;
 
   uint32_t baseCompilerWarmUpThreshold() const {
     switch (level_) {
       case OptimizationLevel::Normal:
         return JitOptions.normalIonWarmUpThreshold;
+      case OptimizationLevel::Full:
+        return JitOptions.fullIonWarmUpThreshold;
       case OptimizationLevel::DontCompile:
       case OptimizationLevel::Wasm:
       case OptimizationLevel::Count:
         break;
     }
     MOZ_CRASH("Unexpected optimization level");
   }
 
@@ -156,31 +184,34 @@ class OptimizationInfo {
         inliningMaxCallerBytecodeLength_(0),
         maxInlineDepth_(0),
         scalarReplacement_(false),
         smallFunctionMaxInlineDepth_(0),
         inliningWarmUpThresholdFactor_(0.0),
         inliningRecompileThresholdFactor_(0) {}
 
   void initNormalOptimizationInfo();
+  void initFullOptimizationInfo();
   void initWasmOptimizationInfo();
 
   OptimizationLevel level() const { return level_; }
 
   bool inlineInterpreted() const {
     return inlineInterpreted_ && !JitOptions.disableInlining;
   }
 
   bool inlineNative() const {
     return inlineNative_ && !JitOptions.disableInlining;
   }
 
   uint32_t compilerWarmUpThreshold(JSScript* script,
                                    jsbytecode* pc = nullptr) const;
 
+  uint32_t recompileWarmUpThreshold(JSScript* script, jsbytecode* pc) const;
+
   bool gvnEnabled() const { return gvn_ && !JitOptions.disableGvn; }
 
   bool licmEnabled() const { return licm_ && !JitOptions.disableLicm; }
 
   bool rangeAnalysisEnabled() const {
     return rangeAnalysis_ && !JitOptions.disableRangeAnalysis;
   }
 
--- a/js/src/jit/MIR.h
+++ b/js/src/jit/MIR.h
@@ -10972,59 +10972,80 @@ class MCheckReturn : public MBinaryInstr
   NAMED_OPERANDS((0, returnValue), (1, thisValue))
 };
 
 // Increase the warm-up counter of the provided script upon execution and test
 // if the warm-up counter surpasses the threshold. Upon hit it will recompile
 // the outermost script (i.e. not the inlined script).
 class MRecompileCheck : public MNullaryInstruction {
  public:
-  enum RecompileCheckType {
-    RecompileCheck_OptimizationLevel,
-    RecompileCheck_Inlining
+  enum class RecompileCheckType : uint8_t {
+    // If we're not at the highest optimization level, keep incrementing the
+    // warm-up counter for the outermost script on entry. The warmup check will
+    // trigger recompilation to tier up. The lazy link mechanism will be used to
+    // tier up once recompilation is done.
+    OptimizationLevel,
+
+    // If we're not at the highest optimization level, keep incrementing the
+    // warm-up counter at loop edges. This check will trigger invalidation for
+    // very long-running loops to ensure we still tier up even if we don't
+    // invoke the lazy link stub.
+    OptimizationLevelOSR,
+
+    // If we're not at the highest optimization level, keep incrementing the
+    // warm-up counter for inlined scripts. This check does not trigger any
+    // recompilation or invalidation, it exists to ensure inlined scripts have
+    // an accurate warm-up count.
+    OptimizationLevelInlined,
+
+    // Used at the last optimization level for callees that weren't hot enough
+    // to be inlined. If a callee becomes hot enough we force recompilation of
+    // the caller's Ion script.
+    Inlining
   };
 
  private:
   JSScript* script_;
   uint32_t recompileThreshold_;
-  bool forceRecompilation_;
-  bool increaseWarmUpCounter_;
+  RecompileCheckType type_;
 
   MRecompileCheck(JSScript* script, uint32_t recompileThreshold,
                   RecompileCheckType type)
       : MNullaryInstruction(classOpcode),
         script_(script),
-        recompileThreshold_(recompileThreshold) {
-    switch (type) {
-      case RecompileCheck_OptimizationLevel:
-        forceRecompilation_ = false;
-        increaseWarmUpCounter_ = true;
-        break;
-      case RecompileCheck_Inlining:
-        forceRecompilation_ = true;
-        increaseWarmUpCounter_ = false;
-        break;
-      default:
-        MOZ_CRASH("Unexpected recompile check type");
-    }
-
+        recompileThreshold_(recompileThreshold),
+        type_(type) {
     setGuard();
   }
 
  public:
   INSTRUCTION_HEADER(RecompileCheck)
   TRIVIAL_NEW_WRAPPERS
 
   JSScript* script() const { return script_; }
 
   uint32_t recompileThreshold() const { return recompileThreshold_; }
 
-  bool forceRecompilation() const { return forceRecompilation_; }
-
-  bool increaseWarmUpCounter() const { return increaseWarmUpCounter_; }
+  bool forceInvalidation() const {
+    return type_ == RecompileCheckType::OptimizationLevelOSR;
+  }
+
+  bool forceRecompilation() const {
+    return type_ == RecompileCheckType::Inlining;
+  }
+
+  bool checkCounter() const {
+    return type_ != RecompileCheckType::OptimizationLevelInlined;
+  }
+
+  bool increaseWarmUpCounter() const {
+    return (type_ == RecompileCheckType::OptimizationLevel ||
+            type_ == RecompileCheckType::OptimizationLevelInlined ||
+            type_ == RecompileCheckType::OptimizationLevelOSR);
+  }
 
   AliasSet getAliasSet() const override { return AliasSet::None(); }
 };
 
 class MAtomicIsLockFree : public MUnaryInstruction,
                           public ConvertToInt32Policy<0>::Data {
   explicit MAtomicIsLockFree(MDefinition* value)
       : MUnaryInstruction(classOpcode, value) {
--- a/js/src/jit/VMFunctionList-inl.h
+++ b/js/src/jit/VMFunctionList-inl.h
@@ -133,16 +133,17 @@ namespace jit {
   _(InterpretResume, js::jit::InterpretResume)                                 \
   _(InterruptCheck, js::jit::InterruptCheck)                                   \
   _(InvokeFunction, js::jit::InvokeFunction)                                   \
   _(InvokeFunctionShuffleNewTarget, js::jit::InvokeFunctionShuffleNewTarget)   \
   _(IonBinaryArithICUpdate, js::jit::IonBinaryArithIC::update)                 \
   _(IonBindNameICUpdate, js::jit::IonBindNameIC::update)                       \
   _(IonCompareICUpdate, js::jit::IonCompareIC::update)                         \
   _(IonCompileScriptForBaseline, js::jit::IonCompileScriptForBaseline)         \
+  _(IonForcedInvalidation, js::jit::IonForcedInvalidation)                     \
   _(IonForcedRecompile, js::jit::IonForcedRecompile)                           \
   _(IonGetIteratorICUpdate, js::jit::IonGetIteratorIC::update)                 \
   _(IonGetNameICUpdate, js::jit::IonGetNameIC::update)                         \
   _(IonGetPropSuperICUpdate, js::jit::IonGetPropSuperIC::update)               \
   _(IonGetPropertyICUpdate, js::jit::IonGetPropertyIC::update)                 \
   _(IonHasOwnICUpdate, js::jit::IonHasOwnIC::update)                           \
   _(IonInICUpdate, js::jit::IonInIC::update)                                   \
   _(IonInstanceOfICUpdate, js::jit::IonInstanceOfIC::update)                   \
--- a/js/src/jit/VMFunctions.cpp
+++ b/js/src/jit/VMFunctions.cpp
@@ -1289,16 +1289,37 @@ bool RecompileImpl(JSContext* cx, bool f
 bool IonForcedRecompile(JSContext* cx) {
   return RecompileImpl(cx, /* force = */ true);
 }
 
 bool IonRecompile(JSContext* cx) {
   return RecompileImpl(cx, /* force = */ false);
 }
 
+bool IonForcedInvalidation(JSContext* cx) {
+  MOZ_ASSERT(cx->currentlyRunningInJit());
+  JitActivationIterator activations(cx);
+  JSJitFrameIter frame(activations->asJit());
+
+  MOZ_ASSERT(frame.type() == FrameType::Exit);
+  ++frame;
+
+  RootedScript script(cx, frame.script());
+  MOZ_ASSERT(script->hasIonScript());
+
+  if (script->baselineScript()->hasPendingIonBuilder()) {
+    LinkIonScript(cx, script);
+    return true;
+  }
+
+  Invalidate(cx, script, /* resetUses = */ false,
+             /* cancelOffThread = */ false);
+  return true;
+}
+
 bool SetDenseElement(JSContext* cx, HandleNativeObject obj, int32_t index,
                      HandleValue value, bool strict) {
   // This function is called from Ion code for StoreElementHole's OOL path.
   // In this case we know the object is native and that no type changes are
   // needed.
 
   DenseElementResult result = obj->setOrExtendDenseElements(
       cx, index, value.address(), 1, ShouldUpdateTypes::DontUpdate);
--- a/js/src/jit/VMFunctions.h
+++ b/js/src/jit/VMFunctions.h
@@ -988,16 +988,18 @@ MOZ_MUST_USE bool InitBaselineFrameForOs
                                           InterpreterFrame* interpFrame,
                                           uint32_t numStackValues);
 
 JSObject* CreateDerivedTypedObj(JSContext* cx, HandleObject descr,
                                 HandleObject owner, int32_t offset);
 
 MOZ_MUST_USE bool IonRecompile(JSContext* cx);
 MOZ_MUST_USE bool IonForcedRecompile(JSContext* cx);
+MOZ_MUST_USE bool IonForcedInvalidation(JSContext* cx);
+
 JSString* StringReplace(JSContext* cx, HandleString string,
                         HandleString pattern, HandleString repl);
 
 MOZ_MUST_USE bool SetDenseElement(JSContext* cx, HandleNativeObject obj,
                                   int32_t index, HandleValue value,
                                   bool strict);
 
 void AssertValidObjectPtr(JSContext* cx, JSObject* obj);