Reland "[osr] Extract extended OSR checks to BaselineOnStackReplacement builtin"

This is a reland of commit a4216b7b

Original change's description:
> [osr] Extract extended OSR checks to BaselineOnStackReplacement builtin
>
> .. to reduce Sparkplug code size.
>
> Bug: v8:12161
> Change-Id: I4029a75dfa37f716c285ce27153c077a0a82a341
> Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3576119
> Reviewed-by: Leszek Swirski <leszeks@chromium.org>
> Commit-Queue: Jakob Linke <jgruber@chromium.org>
> Cr-Commit-Position: refs/heads/main@{#79962}

Bug: v8:12161
Change-Id: I69afd0832d7ca447b5481651ef47ebaa8d023ded
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3585943
Auto-Submit: Jakob Linke <jgruber@chromium.org>
Reviewed-by: Leszek Swirski <leszeks@chromium.org>
Commit-Queue: Leszek Swirski <leszeks@chromium.org>
Cr-Commit-Position: refs/heads/main@{#79980}

src/baseline/baseline-compiler.cc

@@ -1925,50 +1925,30 @@ void BaselineCompiler::VisitCreateRestParameter() {
 }
 
 void BaselineCompiler::VisitJumpLoop() {
-  Label osr_not_armed, osr;
+  Label osr_not_armed;
   {
-    BaselineAssembler::ScratchRegisterScope scope(&basm_);
-    Register osr_urgency_and_install_target = scope.AcquireScratch();
-
     ASM_CODE_COMMENT_STRING(&masm_, "OSR Check Armed");
+    using D = BaselineOnStackReplacementDescriptor;
+    Register osr_urgency_and_install_target =
+        D::OsrUrgencyAndInstallTargetRegister();
     __ LoadRegister(osr_urgency_and_install_target,
                     interpreter::Register::bytecode_array());
     __ LoadWord16FieldZeroExtend(
         osr_urgency_and_install_target, osr_urgency_and_install_target,
         BytecodeArray::kOsrUrgencyAndInstallTargetOffset);
-    int loop_depth = iterator().GetImmediateOperand(1);
+    const int loop_depth = iterator().GetImmediateOperand(1);
     __ JumpIfImmediate(Condition::kUnsignedLessThanEqual,
                        osr_urgency_and_install_target, loop_depth,
                        &osr_not_armed, Label::kNear);
 
-    // TODO(jgruber): Move the extended checks into the
-    // BaselineOnStackReplacement builtin.
-
-    // OSR based on urgency, i.e. is the OSR urgency greater than the current
-    // loop depth?
-    STATIC_ASSERT(BytecodeArray::OsrUrgencyBits::kShift == 0);
-    Register scratch2 = scope.AcquireScratch();
-    __ Word32And(scratch2, osr_urgency_and_install_target,
-                 BytecodeArray::OsrUrgencyBits::kMask);
-    __ JumpIfImmediate(Condition::kUnsignedGreaterThan, scratch2, loop_depth,
-                       &osr, Label::kNear);
-
-    // OSR based on the install target offset, i.e. does the current bytecode
-    // offset match the install target offset?
-    static constexpr int kShift = BytecodeArray::OsrInstallTargetBits::kShift;
-    static constexpr int kMask = BytecodeArray::OsrInstallTargetBits::kMask;
     const int encoded_current_offset =
         BytecodeArray::OsrInstallTargetFor(
             BytecodeOffset{iterator().current_offset()})
-        << kShift;
-    __ Word32And(scratch2, osr_urgency_and_install_target, kMask);
-    __ JumpIfImmediate(Condition::kNotEqual, scratch2, encoded_current_offset,
-                       &osr_not_armed, Label::kNear);
+        << BytecodeArray::OsrInstallTargetBits::kShift;
+    CallBuiltin<Builtin::kBaselineOnStackReplacement>(
+        loop_depth, encoded_current_offset, osr_urgency_and_install_target);
   }
 
-  __ Bind(&osr);
-  CallBuiltin<Builtin::kBaselineOnStackReplacement>();
-
   __ Bind(&osr_not_armed);
   Label* label = &labels_[iterator().GetJumpTargetOffset()]->unlinked;
   int weight = iterator().GetRelativeJumpTargetOffset() -

src/builtins/arm/builtins-arm.cc

@@ -1816,7 +1816,44 @@ void Generate_OSREntry(MacroAssembler* masm, Register entry_address,
   __ Ret();
 }
 
-void OnStackReplacement(MacroAssembler* masm, bool is_interpreter) {
+enum class OsrSourceTier {
+  kInterpreter,
+  kBaseline,
+};
+
+void OnStackReplacement(MacroAssembler* masm, OsrSourceTier source,
+                        Register current_loop_depth,
+                        Register encoded_current_bytecode_offset,
+                        Register osr_urgency_and_install_target) {
+  static constexpr Register scratch = r3;
+  DCHECK(!AreAliased(scratch, current_loop_depth,
+                     encoded_current_bytecode_offset,
+                     osr_urgency_and_install_target));
+  // OSR based on urgency, i.e. is the OSR urgency greater than the current
+  // loop depth?
+  Label try_osr;
+  STATIC_ASSERT(BytecodeArray::OsrUrgencyBits::kShift == 0);
+  Register urgency = scratch;
+  __ and_(urgency, osr_urgency_and_install_target,
+          Operand(BytecodeArray::OsrUrgencyBits::kMask));
+  __ cmp(urgency, current_loop_depth);
+  __ b(hi, &try_osr);
+
+  // OSR based on the install target offset, i.e. does the current bytecode
+  // offset match the install target offset?
+  static constexpr int kMask = BytecodeArray::OsrInstallTargetBits::kMask;
+  Register install_target = osr_urgency_and_install_target;
+  __ and_(install_target, osr_urgency_and_install_target, Operand(kMask));
+  __ cmp(install_target, encoded_current_bytecode_offset);
+  __ b(eq, &try_osr);
+
+  // Neither urgency nor the install target triggered, return to the caller.
+  // Note: the return value must be nullptr or a valid Code object.
+  __ Move(r0, Operand(0));
+  __ Ret(0);
+
+  __ bind(&try_osr);
+
   ASM_CODE_COMMENT(masm);
   {
     FrameAndConstantPoolScope scope(masm, StackFrame::INTERNAL);
@@ -1831,7 +1868,7 @@ void OnStackReplacement(MacroAssembler* masm, bool is_interpreter) {
 
   __ bind(&skip);
 
-  if (is_interpreter) {
+  if (source == OsrSourceTier::kInterpreter) {
     // Drop the handler frame that is be sitting on top of the actual
     // JavaScript frame. This is the case then OSR is triggered from bytecode.
     __ LeaveFrame(StackFrame::STUB);
@@ -1857,13 +1894,24 @@ void OnStackReplacement(MacroAssembler* masm, bool is_interpreter) {
 }  // namespace
 
 void Builtins::Generate_InterpreterOnStackReplacement(MacroAssembler* masm) {
-  return OnStackReplacement(masm, true);
+  using D = InterpreterOnStackReplacementDescriptor;
+  STATIC_ASSERT(D::kParameterCount == 3);
+  OnStackReplacement(masm, OsrSourceTier::kInterpreter,
+                     D::CurrentLoopDepthRegister(),
+                     D::EncodedCurrentBytecodeOffsetRegister(),
+                     D::OsrUrgencyAndInstallTargetRegister());
 }
 
 void Builtins::Generate_BaselineOnStackReplacement(MacroAssembler* masm) {
+  using D = BaselineOnStackReplacementDescriptor;
+  STATIC_ASSERT(D::kParameterCount == 3);
+
   __ ldr(kContextRegister,
          MemOperand(fp, BaselineFrameConstants::kContextOffset));
-  return OnStackReplacement(masm, false);
+  OnStackReplacement(masm, OsrSourceTier::kBaseline,
+                     D::CurrentLoopDepthRegister(),
+                     D::EncodedCurrentBytecodeOffsetRegister(),
+                     D::OsrUrgencyAndInstallTargetRegister());
 }
 
 // static

src/builtins/arm64/builtins-arm64.cc

@@ -2070,7 +2070,44 @@ void Generate_OSREntry(MacroAssembler* masm, Register entry_address,
   __ Br(x17);
 }
 
-void OnStackReplacement(MacroAssembler* masm, bool is_interpreter) {
+enum class OsrSourceTier {
+  kInterpreter,
+  kBaseline,
+};
+
+void OnStackReplacement(MacroAssembler* masm, OsrSourceTier source,
+                        Register current_loop_depth,
+                        Register encoded_current_bytecode_offset,
+                        Register osr_urgency_and_install_target) {
+  static constexpr Register scratch = x3;
+  DCHECK(!AreAliased(scratch, current_loop_depth,
+                     encoded_current_bytecode_offset,
+                     osr_urgency_and_install_target));
+  // OSR based on urgency, i.e. is the OSR urgency greater than the current
+  // loop depth?
+  Label try_osr;
+  STATIC_ASSERT(BytecodeArray::OsrUrgencyBits::kShift == 0);
+  Register urgency = scratch;
+  __ And(urgency, osr_urgency_and_install_target,
+         BytecodeArray::OsrUrgencyBits::kMask);
+  __ Cmp(urgency, current_loop_depth);
+  __ B(hi, &try_osr);
+
+  // OSR based on the install target offset, i.e. does the current bytecode
+  // offset match the install target offset?
+  static constexpr int kMask = BytecodeArray::OsrInstallTargetBits::kMask;
+  Register install_target = osr_urgency_and_install_target;
+  __ And(install_target, osr_urgency_and_install_target, Operand(kMask));
+  __ Cmp(install_target, encoded_current_bytecode_offset);
+  __ B(eq, &try_osr);
+
+  // Neither urgency nor the install target triggered, return to the caller.
+  // Note: the return value must be nullptr or a valid Code object.
+  __ Move(x0, xzr);
+  __ Ret();
+
+  __ bind(&try_osr);
+
   ASM_CODE_COMMENT(masm);
   {
     FrameScope scope(masm, StackFrame::INTERNAL);
@@ -2084,7 +2121,7 @@ void OnStackReplacement(MacroAssembler* masm, bool is_interpreter) {
 
   __ Bind(&skip);
 
-  if (is_interpreter) {
+  if (source == OsrSourceTier::kInterpreter) {
     // Drop the handler frame that is be sitting on top of the actual
     // JavaScript frame. This is the case then OSR is triggered from bytecode.
     __ LeaveFrame(StackFrame::STUB);
@@ -2115,13 +2152,24 @@ void OnStackReplacement(MacroAssembler* masm, bool is_interpreter) {
 }  // namespace
 
 void Builtins::Generate_InterpreterOnStackReplacement(MacroAssembler* masm) {
-  return OnStackReplacement(masm, true);
+  using D = InterpreterOnStackReplacementDescriptor;
+  STATIC_ASSERT(D::kParameterCount == 3);
+  OnStackReplacement(masm, OsrSourceTier::kInterpreter,
+                     D::CurrentLoopDepthRegister(),
+                     D::EncodedCurrentBytecodeOffsetRegister(),
+                     D::OsrUrgencyAndInstallTargetRegister());
 }
 
 void Builtins::Generate_BaselineOnStackReplacement(MacroAssembler* masm) {
+  using D = BaselineOnStackReplacementDescriptor;
+  STATIC_ASSERT(D::kParameterCount == 3);
+
   __ ldr(kContextRegister,
          MemOperand(fp, BaselineFrameConstants::kContextOffset));
-  return OnStackReplacement(masm, false);
+  OnStackReplacement(masm, OsrSourceTier::kBaseline,
+                     D::CurrentLoopDepthRegister(),
+                     D::EncodedCurrentBytecodeOffsetRegister(),
+                     D::OsrUrgencyAndInstallTargetRegister());
 }
 
 // static

src/builtins/builtins-definitions.h

@@ -189,11 +189,11 @@ namespace internal {
       InterpreterPushArgsThenConstruct)                                        \
   ASM(InterpreterEnterAtBytecode, Dummy)                                       \
   ASM(InterpreterEnterAtNextBytecode, Dummy)                                   \
-  ASM(InterpreterOnStackReplacement, ContextOnly)                              \
+  ASM(InterpreterOnStackReplacement, InterpreterOnStackReplacement)            \
                                                                                \
   /* Baseline Compiler */                                                      \
   ASM(BaselineOutOfLinePrologue, BaselineOutOfLinePrologue)                    \
-  ASM(BaselineOnStackReplacement, Void)                                        \
+  ASM(BaselineOnStackReplacement, BaselineOnStackReplacement)                  \
   ASM(BaselineLeaveFrame, BaselineLeaveFrame)                                  \
   ASM(BaselineOrInterpreterEnterAtBytecode, Void)                              \
   ASM(BaselineOrInterpreterEnterAtNextBytecode, Void)                          \

src/builtins/ia32/builtins-ia32.cc

@@ -2801,7 +2801,44 @@ void Generate_OSREntry(MacroAssembler* masm, Register entry_address) {
   __ ret(0);
 }
 
-void OnStackReplacement(MacroAssembler* masm, bool is_interpreter) {
+enum class OsrSourceTier {
+  kInterpreter,
+  kBaseline,
+};
+
+void OnStackReplacement(MacroAssembler* masm, OsrSourceTier source,
+                        Register current_loop_depth,
+                        Register encoded_current_bytecode_offset,
+                        Register osr_urgency_and_install_target) {
+  static constexpr Register scratch = edi;
+  DCHECK(!AreAliased(scratch, current_loop_depth,
+                     encoded_current_bytecode_offset,
+                     osr_urgency_and_install_target));
+  // OSR based on urgency, i.e. is the OSR urgency greater than the current
+  // loop depth?
+  Label try_osr;
+  STATIC_ASSERT(BytecodeArray::OsrUrgencyBits::kShift == 0);
+  Register urgency = scratch;
+  __ Move(urgency, osr_urgency_and_install_target);
+  __ and_(urgency, Immediate(BytecodeArray::OsrUrgencyBits::kMask));
+  __ cmp(urgency, current_loop_depth);
+  __ j(above, &try_osr, Label::kNear);
+
+  // OSR based on the install target offset, i.e. does the current bytecode
+  // offset match the install target offset?
+  static constexpr int kMask = BytecodeArray::OsrInstallTargetBits::kMask;
+  Register install_target = osr_urgency_and_install_target;
+  __ and_(install_target, Immediate(kMask));
+  __ cmp(install_target, encoded_current_bytecode_offset);
+  __ j(equal, &try_osr, Label::kNear);
+
+  // Neither urgency nor the install target triggered, return to the caller.
+  // Note: the return value must be nullptr or a valid Code object.
+  __ Move(eax, Immediate(0));
+  __ ret(0);
+
+  __ bind(&try_osr);
+
   ASM_CODE_COMMENT(masm);
   {
     FrameScope scope(masm, StackFrame::INTERNAL);
@@ -2816,7 +2853,7 @@ void OnStackReplacement(MacroAssembler* masm, bool is_interpreter) {
 
   __ bind(&skip);
 
-  if (is_interpreter) {
+  if (source == OsrSourceTier::kInterpreter) {
     // Drop the handler frame that is be sitting on top of the actual
     // JavaScript frame. This is the case then OSR is triggered from bytecode.
     __ leave();
@@ -2841,13 +2878,24 @@ void OnStackReplacement(MacroAssembler* masm, bool is_interpreter) {
 }  // namespace
 
 void Builtins::Generate_InterpreterOnStackReplacement(MacroAssembler* masm) {
-  return OnStackReplacement(masm, true);
+  using D = InterpreterOnStackReplacementDescriptor;
+  STATIC_ASSERT(D::kParameterCount == 3);
+  OnStackReplacement(masm, OsrSourceTier::kInterpreter,
+                     D::CurrentLoopDepthRegister(),
+                     D::EncodedCurrentBytecodeOffsetRegister(),
+                     D::OsrUrgencyAndInstallTargetRegister());
 }
 
 void Builtins::Generate_BaselineOnStackReplacement(MacroAssembler* masm) {
+  using D = BaselineOnStackReplacementDescriptor;
+  STATIC_ASSERT(D::kParameterCount == 3);
+
   __ mov(kContextRegister,
          MemOperand(ebp, BaselineFrameConstants::kContextOffset));
-  return OnStackReplacement(masm, false);
+  OnStackReplacement(masm, OsrSourceTier::kBaseline,
+                     D::CurrentLoopDepthRegister(),
+                     D::EncodedCurrentBytecodeOffsetRegister(),
+                     D::OsrUrgencyAndInstallTargetRegister());
 }
 
 #if V8_ENABLE_WEBASSEMBLY

src/builtins/x64/builtins-x64.cc

@@ -2729,7 +2729,38 @@ enum class OsrSourceTier {
   kBaseline,
 };
 
-void OnStackReplacement(MacroAssembler* masm, OsrSourceTier source) {
+void OnStackReplacement(MacroAssembler* masm, OsrSourceTier source,
+                        Register current_loop_depth,
+                        Register encoded_current_bytecode_offset,
+                        Register osr_urgency_and_install_target) {
+  DCHECK(!AreAliased(kScratchRegister, current_loop_depth,
+                     encoded_current_bytecode_offset,
+                     osr_urgency_and_install_target));
+  // OSR based on urgency, i.e. is the OSR urgency greater than the current
+  // loop depth?
+  Label try_osr;
+  STATIC_ASSERT(BytecodeArray::OsrUrgencyBits::kShift == 0);
+  Register urgency = kScratchRegister;
+  __ Move(urgency, osr_urgency_and_install_target);
+  __ andq(urgency, Immediate(BytecodeArray::OsrUrgencyBits::kMask));
+  __ cmpq(urgency, current_loop_depth);
+  __ j(above, &try_osr, Label::kNear);
+
+  // OSR based on the install target offset, i.e. does the current bytecode
+  // offset match the install target offset?
+  static constexpr int kMask = BytecodeArray::OsrInstallTargetBits::kMask;
+  Register install_target = osr_urgency_and_install_target;
+  __ andq(install_target, Immediate(kMask));
+  __ cmpq(install_target, encoded_current_bytecode_offset);
+  __ j(equal, &try_osr, Label::kNear);
+
+  // Neither urgency nor the install target triggered, return to the caller.
+  // Note: the return value must be nullptr or a valid Code object.
+  __ Move(rax, Immediate(0));
+  __ ret(0);
+
+  __ bind(&try_osr);
+
   {
     FrameScope scope(masm, StackFrame::INTERNAL);
     __ CallRuntime(Runtime::kCompileOptimizedOSR);
@@ -2771,13 +2802,24 @@ void OnStackReplacement(MacroAssembler* masm, OsrSourceTier source) {
 }  // namespace
 
 void Builtins::Generate_InterpreterOnStackReplacement(MacroAssembler* masm) {
-  OnStackReplacement(masm, OsrSourceTier::kInterpreter);
+  using D = InterpreterOnStackReplacementDescriptor;
+  STATIC_ASSERT(D::kParameterCount == 3);
+  OnStackReplacement(masm, OsrSourceTier::kInterpreter,
+                     D::CurrentLoopDepthRegister(),
+                     D::EncodedCurrentBytecodeOffsetRegister(),
+                     D::OsrUrgencyAndInstallTargetRegister());
 }
 
 void Builtins::Generate_BaselineOnStackReplacement(MacroAssembler* masm) {
+  using D = BaselineOnStackReplacementDescriptor;
+  STATIC_ASSERT(D::kParameterCount == 3);
+
   __ movq(kContextRegister,
           MemOperand(rbp, BaselineFrameConstants::kContextOffset));
-  OnStackReplacement(masm, OsrSourceTier::kBaseline);
+  OnStackReplacement(masm, OsrSourceTier::kBaseline,
+                     D::CurrentLoopDepthRegister(),
+                     D::EncodedCurrentBytecodeOffsetRegister(),
+                     D::OsrUrgencyAndInstallTargetRegister());
 }
 
 #if V8_ENABLE_WEBASSEMBLY

src/codegen/interface-descriptors-inl.h

@@ -356,6 +356,52 @@ constexpr auto BaselineLeaveFrameDescriptor::registers() {
 #endif
 }
 
+// static
+constexpr auto BaselineOnStackReplacementDescriptor::registers() {
+  return DefaultRegisterArray();
+}
+
+// static
+constexpr Register
+BaselineOnStackReplacementDescriptor::CurrentLoopDepthRegister() {
+  return registers()[0];
+}
+// static
+constexpr Register
+BaselineOnStackReplacementDescriptor::EncodedCurrentBytecodeOffsetRegister() {
+  return registers()[1];
+}
+// static
+constexpr Register
+BaselineOnStackReplacementDescriptor::OsrUrgencyAndInstallTargetRegister() {
+  return registers()[2];
+}
+
+// static
+constexpr auto InterpreterOnStackReplacementDescriptor::registers() {
+  using BaselineD = BaselineOnStackReplacementDescriptor;
+  return BaselineD::registers();
+}
+
+// static
+constexpr Register
+InterpreterOnStackReplacementDescriptor::CurrentLoopDepthRegister() {
+  using BaselineD = BaselineOnStackReplacementDescriptor;
+  return BaselineD::CurrentLoopDepthRegister();
+}
+// static
+constexpr Register InterpreterOnStackReplacementDescriptor::
+    EncodedCurrentBytecodeOffsetRegister() {
+  using BaselineD = BaselineOnStackReplacementDescriptor;
+  return BaselineD::EncodedCurrentBytecodeOffsetRegister();
+}
+// static
+constexpr Register
+InterpreterOnStackReplacementDescriptor::OsrUrgencyAndInstallTargetRegister() {
+  using BaselineD = BaselineOnStackReplacementDescriptor;
+  return BaselineD::OsrUrgencyAndInstallTargetRegister();
+}
+
 // static
 constexpr auto VoidDescriptor::registers() { return RegisterArray(); }
 

src/codegen/interface-descriptors.h

@@ -31,16 +31,17 @@ namespace internal {
   V(ArrayNoArgumentConstructor)                      \
   V(ArraySingleArgumentConstructor)                  \
   V(AsyncFunctionStackParameter)                     \
+  V(BaselineLeaveFrame)                              \
+  V(BaselineOnStackReplacement)                      \
+  V(BaselineOutOfLinePrologue)                       \
   V(BigIntToI32Pair)                                 \
   V(BigIntToI64)                                     \
   V(BinaryOp)                                        \
   V(BinaryOp_Baseline)                               \
-  V(BinarySmiOp_Baseline)                            \
   V(BinaryOp_WithFeedback)                           \
+  V(BinarySmiOp_Baseline)                            \
   V(CallForwardVarargs)                              \
   V(CallFunctionTemplate)                            \
-  V(CopyDataPropertiesWithExcludedProperties)        \
-  V(CopyDataPropertiesWithExcludedPropertiesOnStack) \
   V(CallTrampoline)                                  \
   V(CallTrampoline_Baseline)                         \
   V(CallTrampoline_Baseline_Compact)                 \
@@ -57,17 +58,19 @@ namespace internal {
   V(Compare)                                         \
   V(Compare_Baseline)                                \
   V(Compare_WithFeedback)                            \
+  V(Construct_Baseline)                              \
   V(ConstructForwardVarargs)                         \
   V(ConstructStub)                                   \
   V(ConstructVarargs)                                \
   V(ConstructWithArrayLike)                          \
   V(ConstructWithArrayLike_WithFeedback)             \
   V(Construct_WithFeedback)                          \
-  V(Construct_Baseline)                              \
   V(ConstructWithSpread)                             \
   V(ConstructWithSpread_Baseline)                    \
   V(ConstructWithSpread_WithFeedback)                \
   V(ContextOnly)                                     \
+  V(CopyDataPropertiesWithExcludedProperties)        \
+  V(CopyDataPropertiesWithExcludedPropertiesOnStack) \
   V(CppBuiltinAdaptor)                               \
   V(FastNewObject)                                   \
   V(ForInPrepare)                                    \
@@ -79,11 +82,15 @@ namespace internal {
   V(InterpreterCEntry1)                              \
   V(InterpreterCEntry2)                              \
   V(InterpreterDispatch)                             \
+  V(InterpreterOnStackReplacement)                   \
   V(InterpreterPushArgsThenCall)                     \
   V(InterpreterPushArgsThenConstruct)                \
   V(JSTrampoline)                                    \
-  V(BaselineOutOfLinePrologue)                       \
-  V(BaselineLeaveFrame)                              \
+  V(KeyedHasICBaseline)                              \
+  V(KeyedHasICWithVector)                            \
+  V(KeyedLoad)                                       \
+  V(KeyedLoadBaseline)                               \
+  V(KeyedLoadWithVector)                             \
   V(Load)                                            \
   V(LoadBaseline)                                    \
   V(LoadGlobal)                                      \
@@ -91,18 +98,12 @@ namespace internal {
   V(LoadGlobalNoFeedback)                            \
   V(LoadGlobalWithVector)                            \
   V(LoadNoFeedback)                                  \
-  V(LoadWithVector)                                  \
-  V(KeyedLoad)                                       \
-  V(KeyedLoadBaseline)                               \
-  V(KeyedLoadWithVector)                             \
-  V(KeyedHasICBaseline)                              \
-  V(KeyedHasICWithVector)                            \
   V(LoadWithReceiverAndVector)                       \
   V(LoadWithReceiverBaseline)                        \
+  V(LoadWithVector)                                  \
   V(LookupBaseline)                                  \
   V(NoContext)                                       \
   V(ResumeGenerator)                                 \
-  V(SuspendGeneratorBaseline)                        \
   V(ResumeGeneratorBaseline)                         \
   V(RunMicrotasks)                                   \
   V(RunMicrotasksEntry)                              \
@@ -116,11 +117,10 @@ namespace internal {
   V(StoreWithVector)                                 \
   V(StringAtAsString)                                \
   V(StringSubstring)                                 \
-  IF_TSAN(V, TSANStore)                              \
-  IF_TSAN(V, TSANLoad)                               \
+  V(SuspendGeneratorBaseline)                        \
   V(TypeConversion)                                  \
-  V(TypeConversionNoContext)                         \
   V(TypeConversion_Baseline)                         \
+  V(TypeConversionNoContext)                         \
   V(Typeof)                                          \
   V(UnaryOp_Baseline)                                \
   V(UnaryOp_WithFeedback)                            \
@@ -131,6 +131,8 @@ namespace internal {
   V(WasmI64AtomicWait32)                             \
   V(WasmSuspend)                                     \
   V(WriteBarrier)                                    \
+  IF_TSAN(V, TSANLoad)                               \
+  IF_TSAN(V, TSANStore)                              \
   BUILTIN_LIST_TFS(V)                                \
   TORQUE_BUILTIN_LIST_TFC(V)
 
@@ -1688,6 +1690,42 @@ class BaselineLeaveFrameDescriptor
   static constexpr inline auto registers();
 };
 
+class InterpreterOnStackReplacementDescriptor
+    : public StaticCallInterfaceDescriptor<
+          InterpreterOnStackReplacementDescriptor> {
+ public:
+  DEFINE_PARAMETERS(kCurrentLoopDepth, kEncodedCurrentBytecodeOffset,
+                    kOsrUrgencyAndInstallTarget)
+  DEFINE_PARAMETER_TYPES(MachineType::Int32(),  // kCurrentLoopDepth
+                         MachineType::Int32(),  // kEncodedCurrentBytecodeOffset
+                         MachineType::Int32())  // kOsrUrgencyAndInstallTarget
+  DECLARE_DESCRIPTOR(InterpreterOnStackReplacementDescriptor)
+
+  static constexpr inline Register CurrentLoopDepthRegister();
+  static constexpr inline Register EncodedCurrentBytecodeOffsetRegister();
+  static constexpr inline Register OsrUrgencyAndInstallTargetRegister();
+
+  static constexpr inline auto registers();
+};
+
+class BaselineOnStackReplacementDescriptor
+    : public StaticCallInterfaceDescriptor<
+          BaselineOnStackReplacementDescriptor> {
+ public:
+  DEFINE_PARAMETERS_NO_CONTEXT(kCurrentLoopDepth, kEncodedCurrentBytecodeOffset,
+                               kOsrUrgencyAndInstallTarget)
+  DEFINE_PARAMETER_TYPES(MachineType::Int32(),  // kCurrentLoopDepth
+                         MachineType::Int32(),  // kEncodedCurrentBytecodeOffset
+                         MachineType::Int32())  // kOsrUrgencyAndInstallTarget
+  DECLARE_DESCRIPTOR(BaselineOnStackReplacementDescriptor)
+
+  static constexpr inline Register CurrentLoopDepthRegister();
+  static constexpr inline Register EncodedCurrentBytecodeOffsetRegister();
+  static constexpr inline Register OsrUrgencyAndInstallTargetRegister();
+
+  static constexpr inline auto registers();
+};
+
 class V8_EXPORT_PRIVATE InterpreterDispatchDescriptor
     : public StaticCallInterfaceDescriptor<InterpreterDispatchDescriptor> {
  public:

src/codegen/register.h

@@ -19,18 +19,17 @@ constexpr bool ShouldPadArguments(int argument_count) {
   return ArgumentPaddingSlots(argument_count) != 0;
 }
 
-#ifdef DEBUG
 template <typename... RegTypes,
           // All arguments must be either Register or DoubleRegister.
           typename = typename std::enable_if_t<
               std::conjunction_v<std::is_same<Register, RegTypes>...> ||
               std::conjunction_v<std::is_same<DoubleRegister, RegTypes>...>>>
 inline constexpr bool AreAliased(RegTypes... regs) {
-  int num_different_regs = RegListBase{regs...}.Count();
+  using FirstRegType = std::tuple_element_t<0, std::tuple<RegTypes...>>;
+  int num_different_regs = RegListBase<FirstRegType>{regs...}.Count();
   int num_given_regs = (... + (regs.is_valid() ? 1 : 0));
   return num_different_regs < num_given_regs;
 }
-#endif
 
 }  // namespace internal
 }  // namespace v8

src/heap/factory.cc

@@ -2415,7 +2415,8 @@ Handle<BytecodeArray> Factory::CopyBytecodeArray(Handle<BytecodeArray> source) {
   copy.set_handler_table(raw_source.handler_table());
   copy.set_source_position_table(raw_source.source_position_table(kAcquireLoad),
                                  kReleaseStore);
-  copy.set_osr_urgency(raw_source.osr_urgency());
+  copy.set_osr_urgency_and_install_target(
+      raw_source.osr_urgency_and_install_target());
   copy.set_bytecode_age(raw_source.bytecode_age());
   raw_source.CopyBytecodesTo(copy);
   return handle(copy, isolate());

src/interpreter/interpreter-assembler.cc

@@ -1339,19 +1339,39 @@ void InterpreterAssembler::AbortIfWordNotEqual(TNode<WordT> lhs,
   BIND(&ok);
 }
 
-void InterpreterAssembler::OnStackReplacement(TNode<Context> context,
-                                              TNode<IntPtrT> relative_jump) {
-  TNode<JSFunction> function = CAST(LoadRegister(Register::function_closure()));
-  TNode<HeapObject> shared_info = LoadJSFunctionSharedFunctionInfo(function);
-  TNode<Object> sfi_data =
-      LoadObjectField(shared_info, SharedFunctionInfo::kFunctionDataOffset);
-  TNode<Uint16T> data_type = LoadInstanceType(CAST(sfi_data));
+void InterpreterAssembler::OnStackReplacement(
+    TNode<Context> context, TNode<IntPtrT> relative_jump,
+    TNode<Int32T> loop_depth, TNode<Int16T> osr_urgency_and_install_target) {
+  Label interpreter(this), baseline(this);
+  {
+    TNode<JSFunction> function =
+        CAST(LoadRegister(Register::function_closure()));
+    TNode<HeapObject> shared_info = LoadJSFunctionSharedFunctionInfo(function);
+    TNode<Object> sfi_data =
+        LoadObjectField(shared_info, SharedFunctionInfo::kFunctionDataOffset);
+    TNode<Uint16T> data_type = LoadInstanceType(CAST(sfi_data));
+    Branch(InstanceTypeEqual(data_type, CODET_TYPE), &baseline, &interpreter);
+  }
 
-  Label baseline(this);
-  GotoIf(InstanceTypeEqual(data_type, CODET_TYPE), &baseline);
+  BIND(&interpreter);
   {
+    // Encode the current bytecode offset as
+    //
+    //  BytecodeArray::OsrInstallTargetFor(
+    //      BytecodeOffset{iterator().current_offset()})
+    //  << BytecodeArray::OsrInstallTargetBits::kShift
+    static constexpr int kShift = BytecodeArray::OsrInstallTargetBits::kShift;
+    static constexpr int kMask = BytecodeArray::OsrInstallTargetBits::kMask;
+    TNode<Word32T> encoded_bytecode_offset = Word32Or(
+        Int32Sub(TruncateIntPtrToInt32(BytecodeOffset()), kFirstBytecodeOffset),
+        Int32Constant(1));
+    encoded_bytecode_offset = Word32And(
+        Word32Shl(UncheckedCast<Int32T>(encoded_bytecode_offset), kShift),
+        kMask);
+
     Callable callable = CodeFactory::InterpreterOnStackReplacement(isolate());
-    CallStub(callable, context);
+    CallStub(callable, context, loop_depth, encoded_bytecode_offset,
+             osr_urgency_and_install_target);
     JumpBackward(relative_jump);
   }
 

src/interpreter/interpreter-assembler.h

@@ -263,8 +263,12 @@ class V8_EXPORT_PRIVATE InterpreterAssembler : public CodeStubAssembler {
       TNode<FixedArrayBase> parameters_and_registers,
       TNode<IntPtrT> formal_parameter_count, TNode<UintPtrT> register_count);
 
-  // Perform OnStackReplacement.
-  void OnStackReplacement(TNode<Context> context, TNode<IntPtrT> relative_jump);
+  // Attempts to OSR; note this may fail in some cases, e.g. on a mismatched
+  // install target, or if there's no compiled code object available yet
+  // (concurrent OSR).
+  void OnStackReplacement(TNode<Context> context, TNode<IntPtrT> relative_jump,
+                          TNode<Int32T> loop_depth,
+                          TNode<Int16T> osr_urgency_and_install_target);
 
   // The BytecodeOffset() is the offset from the ByteCodeArray pointer; to
   // translate into runtime `BytecodeOffset` (defined in utils.h as the offset

src/interpreter/interpreter-generator.cc

@@ -2173,7 +2173,7 @@ IGNITION_HANDLER(JumpLoop, InterpreterAssembler) {
   // OSR requests can be triggered either through urgency (when > the current
   // loop depth), or an explicit install target (= the lower bits of the
   // targeted bytecode offset).
-  Label ok(this), maybe_osr(this, Label::kDeferred);
+  Label ok(this), maybe_osr(this);
   Branch(Int32GreaterThanOrEqual(loop_depth, osr_urgency_and_install_target),
          &ok, &maybe_osr);
 
@@ -2183,30 +2183,8 @@ IGNITION_HANDLER(JumpLoop, InterpreterAssembler) {
   JumpBackward(relative_jump);
 
   BIND(&maybe_osr);
-  Label osr(this);
-  // OSR based on urgency, i.e. is the OSR urgency greater than the current
-  // loop depth?
-  STATIC_ASSERT(BytecodeArray::OsrUrgencyBits::kShift == 0);
-  TNode<Word32T> osr_urgency = Word32And(osr_urgency_and_install_target,
-                                         BytecodeArray::OsrUrgencyBits::kMask);
-  GotoIf(Int32GreaterThan(osr_urgency, loop_depth), &osr);
-
-  // OSR based on the install target offset, i.e. does the current bytecode
-  // offset match the install target offset?
-  //
-  //  if (((offset << kShift) & kMask) == (target & kMask)) { ... }
-  static constexpr int kShift = BytecodeArray::OsrInstallTargetBits::kShift;
-  static constexpr int kMask = BytecodeArray::OsrInstallTargetBits::kMask;
-  // Note: We OR in 1 to avoid 0 offsets, see Code::OsrInstallTargetFor.
-  TNode<Word32T> actual = Word32Or(
-      Int32Sub(TruncateIntPtrToInt32(BytecodeOffset()), kFirstBytecodeOffset),
-      Int32Constant(1));
-  actual = Word32And(Word32Shl(UncheckedCast<Int32T>(actual), kShift), kMask);
-  TNode<Word32T> expected = Word32And(osr_urgency_and_install_target, kMask);
-  Branch(Word32Equal(actual, expected), &osr, &ok);
-
-  BIND(&osr);
-  OnStackReplacement(context, relative_jump);
+  OnStackReplacement(context, relative_jump, loop_depth,
+                     osr_urgency_and_install_target);
 }
 
 // SwitchOnSmiNoFeedback <table_start> <table_length> <case_value_base>

src/snapshot/deserializer.cc

@@ -560,7 +560,7 @@ void Deserializer<IsolateT>::PostProcessNewObject(Handle<Map> map,
   } else if (InstanceTypeChecker::IsBytecodeArray(instance_type)) {
     // TODO(mythria): Remove these once we store the default values for these
     // fields in the serializer.
-    BytecodeArray::cast(raw_obj).reset_osr_urgency();
+    BytecodeArray::cast(raw_obj).reset_osr_urgency_and_install_target();
   } else if (InstanceTypeChecker::IsDescriptorArray(instance_type)) {
     DCHECK(InstanceTypeChecker::IsStrongDescriptorArray(instance_type));
     Handle<DescriptorArray> descriptors = Handle<DescriptorArray>::cast(obj);